diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ce5d935..a8f1c0f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,6 +17,10 @@ updates: time: "09:00" timezone: America/Los_Angeles open-pull-requests-limit: 5 + ignore: + - dependency-name: mcp + update-types: + - version-update:semver-major - package-ecosystem: pip directory: /plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp @@ -26,6 +30,10 @@ updates: time: "09:00" timezone: America/Los_Angeles open-pull-requests-limit: 5 + ignore: + - dependency-name: mcp + update-types: + - version-update:semver-major - package-ecosystem: github-actions directory: / diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c65a19..519d453 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Run public static gates run: make check - name: Check Python syntax - run: python -m compileall -q plugins/profile-and-optimize/server scripts + run: python -m compileall -q plugins/profile-and-optimize scripts - name: Check repository whitespace run: git diff --check 4b825dc642cb6eb9a060e54bf8d69288fbee4904 HEAD @@ -72,6 +72,9 @@ jobs: run: make smoke-test - name: Run tool tests run: make pytest + - name: Type-check production Python + if: matrix.python-version == '3.12' + run: make typecheck - name: Run the MCP protocol smoke test run: make smoke-mcp-runtime diff --git a/.gitignore b/.gitignore index 03905d5..2981ddd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ __pycache__/ .venv/ .venv-test/ *.egg-info/ +build/ +dist/ .env .env.local .idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce28791..2861c59 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,14 +25,25 @@ repos: rev: v0.16.3 hooks: - id: ruff - args: [--fix, --select, E4,E7,E9,F, --config, plugins/profile-and-optimize/server/pyproject.toml] - files: '^plugins/profile-and-optimize/server/(perf_baseline|evidence|slurm|tools/(perf_baseline|evidence|slurm))/' + args: [--fix, --config, plugins/profile-and-optimize/server/pyproject.toml] + files: '^(plugins/profile-and-optimize/|scripts/).*\.py$' - id: ruff-format args: [--config, plugins/profile-and-optimize/server/pyproject.toml] - files: '^plugins/profile-and-optimize/server/(perf_baseline|evidence|slurm|tools/(perf_baseline|evidence|slurm))/' + files: '^(plugins/profile-and-optimize/|scripts/).*\.py$' - repo: local hooks: + - id: pyright-production + name: pyright (production Python) + entry: make typecheck + language: system + pass_filenames: false + files: '(^plugins/profile-and-optimize/(server|hooks|skills)/.*\.py$|^scripts/.*\.py$|^pyrightconfig\.json$)' + - id: shellcheck + name: shellcheck (tracked shell scripts) + entry: bash scripts/check-shell.sh + language: system + types: [shell] - id: check-doc-links-no-network name: check-doc-links (no-network) entry: bash scripts/check-doc-links.sh --no-network --quiet @@ -53,7 +64,7 @@ repos: files: '(^plugins/profile-and-optimize/skills/|^README\.md$|^CONTRIBUTING\.md$|^REVIEWERS\.md$|^SECURITY\.md$|^Makefile$|^plugins/profile-and-optimize/README\.md$|^plugins/profile-and-optimize/\.claude-plugin/plugin\.json$|^\.claude-plugin/marketplace\.json$)' - id: lint-tool-counts name: lint-tool-counts (single source of truth for MCP tool / library totals) - entry: python3 scripts/lint-tool-counts.py + entry: make lint-tool-counts language: system pass_filenames: false files: '(^plugins/profile-and-optimize/server/(mcp_surface\.py|README\.md|AGENTS\.md|docs/|tools/profile_and_optimize_mcp/README\.md)|^README\.md$|^CONTRIBUTING\.md$|^REVIEWERS\.md$|^SECURITY\.md$|^Makefile$|^plugins/profile-and-optimize/README\.md$|^plugins/profile-and-optimize/\.claude-plugin/plugin\.json$)' diff --git a/AGENTS.md b/AGENTS.md index 60986bc..69e0654 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,8 @@ Client-specific files may point here, but shared policy belongs in this file. ```bash make demo # print the tool and skill surface, no GPU needed make help # list operator targets -make all # run smoke checks, doc links, skill lint, and pytest +make quality # run Ruff, Pyright, and ShellCheck +make all # run quality checks, docs, skill lint, MCP smoke, and pytest ``` The repository layout, client setup, and methodology live in @@ -119,8 +120,9 @@ source revisions, or delivery methods. ## Working in this repository -- Run `make all` before committing. Use `make -j4 all` to run independent - targets in parallel. +- Run `make all` before committing. It includes Python lint and formatting, + production type checks, shell lint, repository gates, MCP smoke, and pytest. + Use `make -j4 all` to run independent targets in parallel. - Skill, tool, and library counts are canonical in [`plugins/profile-and-optimize/server/mcp_surface.py`](plugins/profile-and-optimize/server/mcp_surface.py). Never hardcode a replacement count without updating and running the count diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f4857c..bac68a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2026-08-16 + +### Added +- Added repository-wide Ruff, Pyright, and ShellCheck gates to `make all`, + pre-commit, and CI. +- Added an upgrade guide, a Claude Code MCP health check, and a Codex MCP + registration check. + +### Security +- Kept Artificial Analysis API keys out of subprocess arguments, command + receipts, standard output, standard error, CLI JSON, and MCP responses. +- Rejected custom AIPerf commands that embed `--api-key` before they can run or + create evidence artifacts. + +### Changed +- Required an explicit `--client` in the direct MCP installer and client + configuration helper. The installer no longer writes Cursor configuration + when the caller omits a client. +- Updated the pinned Artificial Analysis dependency closure and held the MCP + runtime below version 2 pending a deliberate compatibility migration. + +### Fixed +- Corrected production type contracts across the server, hooks, skills, and + repository scripts. +- Fixed a variable-shadowing bug that could corrupt K-suffixed benchmark cell + identifiers when variant metadata was present. +- Documented the search-tool fallback when `rg` is not installed. + ## [0.3.1] - 2026-08-16 ### Security diff --git a/CLAUDE.md b/CLAUDE.md index 2ae9212..557660c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,3 +9,10 @@ in that adapter. Keep shared project policy in `AGENTS.md`. The shared value bar is that every promoted result must be adversarially-confirmed to add value. + +## Health Stack + +- typecheck: make typecheck +- lint: make lint +- test: make pytest +- shell: make lint-shell diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 609abb3..6c809b0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,10 +15,11 @@ bash plugins/profile-and-optimize/server/install.sh --with-dev make all ``` -The development install includes the official Agent Skills validator, test -tools, and every dependency required by the default test suite. `make all` -runs the client-neutral checks. Use `make -j4 all` to run independent targets -in parallel. +The development install includes the official Agent Skills validator, Ruff, +Pyright, ShellCheck, test tools, and every dependency required by the default +test suite. `make quality` runs the language checks. `make all` adds docs, +skill validation, the MCP smoke test, and pytest. Use `make -j4 all` to run +independent targets in parallel. Claude Code packaging has a separate optional check: diff --git a/Makefile b/Makefile index f38cfcb..163ac2b 100644 --- a/Makefile +++ b/Makefile @@ -10,12 +10,14 @@ SERVER_DIR := $(PLUGIN_DIR)/server SCRIPTS_DIR := scripts SERVER_PY := $(SERVER_DIR)/.venv/bin/python PYTHON ?= $(if $(wildcard $(SERVER_PY)),$(SERVER_PY),python3) +RUFF ?= $(if $(wildcard $(SERVER_DIR)/.venv/bin/ruff),$(SERVER_DIR)/.venv/bin/ruff,ruff) +CURRENT_VERSION := $(strip $(shell cat VERSION 2>/dev/null)) # Default VERSION for `make release-notes`. Override on the command line: -# make release-notes VERSION=v0.3.1 +# make release-notes VERSION=v$(CURRENT_VERSION) VERSION ?= -.PHONY: help demo check validate validate-agent-skills validate-claude-plugin validate-claude-plugin-uncached validate-uncached smoke-test smoke-mcp-runtime check-doc-links workload-proof-check lint-skill-mcp-args lint-skill-counts lint-tool-counts lint-versions check-version-transition test-release-gates pytest pytest-mcp pytest-xdist all freshness bootstrap print-mcp-snippet doctor install-mcp install-skills install-into-codex install-into-cursor refresh-symlinks release release-notes mcp-surface clean-pycache +.PHONY: help demo check quality lint typecheck lint-shell validate validate-agent-skills validate-claude-plugin validate-claude-plugin-uncached validate-uncached smoke-test smoke-mcp-runtime check-doc-links workload-proof-check lint-skill-mcp-args lint-skill-counts lint-tool-counts lint-versions check-version-transition test-release-gates pytest pytest-mcp pytest-xdist all freshness bootstrap print-mcp-snippet doctor install-mcp install-skills install-into-codex install-into-cursor refresh-symlinks release release-notes mcp-surface clean-pycache help: @printf 'profile-and-optimize operator commands\n\n' @@ -23,6 +25,10 @@ help: @printf ' make demo List all skills and a sample of the MCP surface. No GPU required\n' @printf ' make check Run the public static gates, including Agent Skills and all Markdown links\n' @printf ' make all Run the full client-neutral local test suite. Use `make -j4 all` for parallel execution\n' + @printf ' make quality Run Ruff, Pyright, and ShellCheck\n' + @printf ' make lint Check Python lint and formatting with Ruff\n' + @printf ' make typecheck Type-check production Python with Pyright\n' + @printf ' make lint-shell Check every tracked shell script with ShellCheck\n' @printf ' make validate-agent-skills Validate every skill with the official Agent Skills reference validator\n' @printf ' make validate-claude-plugin Run the optional Claude Code package validator\n' @printf ' make smoke-test Validate skills, canonical counts, versions, and MCP argument references\n' @@ -50,11 +56,11 @@ help: @printf ' make bootstrap Set up the best-effort Cursor adapter from a clone\n' @printf ' make print-mcp-snippet Print a Cursor MCP config block without writing it\n' @printf ' make doctor Diagnose a Cursor MCP entry. Read-only unless FIX=1\n' - @printf ' make release-notes VERSION=v0.3.1 Extract the CHANGELOG section for v0.3.1\n' + @printf ' make release-notes VERSION=v%s Extract the CHANGELOG section for the current release\n' '$(CURRENT_VERSION)' @printf ' make clean-pycache Remove __pycache__ + *.pyc under server/\n' @printf '\n' @printf 'Variables (override on command line):\n' - @printf ' VERSION Version tag for release-notes (e.g. v0.3.1)\n' + @printf ' VERSION Version tag for release-notes (e.g. v%s)\n' '$(CURRENT_VERSION)' validate: validate-agent-skills @@ -85,16 +91,41 @@ smoke-test: validate-agent-skills @$(PYTHON) $(SCRIPTS_DIR)/lint-tool-counts.py @echo '--- version-header lint ---' @$(PYTHON) $(SCRIPTS_DIR)/lint-versions.py + @$(PYTHON) $(SCRIPTS_DIR)/extract-release-notes.py "v$(CURRENT_VERSION)" CHANGELOG.md >/dev/null @echo '--- skill MCP argument lint ---' @$(PYTHON) $(SCRIPTS_DIR)/lint-skill-mcp-args.py -check: smoke-test +check: smoke-test lint lint-shell @$(PYTHON) scripts/check-version-transition.py @$(PYTHON) scripts/test_release_gates.py @$(PYTHON) scripts/check_docs.py @$(PYTHON) scripts/check_workload_proof_packets.py --self-test --require-workflow-handoff @bash scripts/check-doc-links.sh --no-network --quiet +quality: lint typecheck lint-shell + @echo '[ok] Python and shell quality checks passed' + +lint: + @if ! command -v "$(RUFF)" >/dev/null 2>&1 && [ ! -x "$(RUFF)" ]; then \ + echo '[FAIL] Ruff is not installed. Run: bash $(SERVER_DIR)/install.sh --with-dev'; \ + exit 2; \ + fi + @$(RUFF) check --config $(SERVER_DIR)/pyproject.toml $(PLUGIN_DIR) $(SCRIPTS_DIR) + @$(RUFF) format --check --config $(SERVER_DIR)/pyproject.toml $(PLUGIN_DIR) $(SCRIPTS_DIR) + +typecheck: + @if [ -x "$(SERVER_DIR)/.venv/bin/pyright" ]; then \ + $(SERVER_DIR)/.venv/bin/pyright --project pyrightconfig.json; \ + elif command -v pyright >/dev/null 2>&1; then \ + pyright --project pyrightconfig.json; \ + else \ + echo '[FAIL] Pyright is not installed. Run: bash $(SERVER_DIR)/install.sh --with-dev'; \ + exit 2; \ + fi + +lint-shell: + @bash $(SCRIPTS_DIR)/check-shell.sh + demo: @printf '== gpu-perf-tune: 32 Agent Skills ==\n' @find $(PLUGIN_DIR)/skills -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort | sed 's/^/ /' @@ -186,7 +217,7 @@ freshness: # These targets are independent and can run in parallel with `make -j4 all`. # The static check includes offline Markdown links. Scheduled CI checks live # external URLs so local validation does not depend on network availability. -all: check smoke-mcp-runtime pytest +all: check typecheck smoke-mcp-runtime pytest @echo '[ok] all client-neutral checks passed' lint-skill-mcp-args: diff --git a/README.md b/README.md index 8e7a310..ba4bf00 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,12 @@ From a clone of this repository: ```bash make install-skills CLIENT=codex make install-mcp CLIENT=codex +codex mcp get profile_and_optimize ``` -Restart Codex after installation. Codex discovers the linked skills under +The registration check should report `enabled: true`. Run +`make smoke-mcp-runtime` for a live server handshake, then restart Codex after +installation. Codex discovers the linked skills under `~/.agents/skills`. Use `/skills`, invoke a skill with `$skill-name`, or describe the task and let Codex select one. The MCP configuration is shared by Codex CLI, the IDE extension, and the desktop app. See the official Codex @@ -107,11 +110,14 @@ claude plugin install --scope user \ # Install the bundled MCP server inside the current plugin cache entry. # Add --full when you need the PDF report dependencies. bash "$(ls -dt ~/.claude/plugins/cache/profile-and-optimize-plugins/profile-and-optimize/*/server/install.sh | head -1)" + +claude mcp get plugin:profile-and-optimize:profile_and_optimize ``` -Restart Claude Code. Invoke a skill such as `/inference-perf-bench`, or describe -the task and let Claude Code select a matching skill. The plugin installs its -provenance hook, but the hook remains inactive until +The health check should report `Status: ✔ Connected`. Restart Claude Code. +Invoke a skill such as `/inference-perf-bench`, or describe the task and let +Claude Code select a matching skill. The plugin installs its provenance hook, +but the hook remains inactive until `PROVENANCE_COMMIT_GATE=ask` or `PROVENANCE_COMMIT_GATE=deny` is present in the Claude hook environment. Install `jq` on `PATH` before enabling either mode. @@ -127,6 +133,12 @@ make install-skills CLIENT=cursor make install-mcp CLIENT=cursor ``` +## Upgrading + +Read [`docs/UPGRADING.md`](docs/UPGRADING.md) before changing release lines. It +lists the client refresh commands and required installer, AI tuning, MLPerf +rules, and MCP acknowledgement changes. + ## Value bar Every benchmark result, optimization claim, and generated report starts as a diff --git a/SECURITY.md b/SECURITY.md index 938c67e..13dcbd5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,8 +7,8 @@ Security fixes land on `main` while the next release is in development. | Version | Status | | --- | --- | | `main` | Supported development branch | -| `0.3.x` | Supported release line | -| `0.2.x` and earlier | Unsupported | +| `0.4.x` | Supported release line | +| `0.3.x` and earlier | Unsupported | The latest published release may not contain fixes that have landed on `main`. Check the [release history](https://github.com/cfregly/gpu-perf-tune/releases) before deploying. diff --git a/VERSION b/VERSION index 9e11b32..1d0ba9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 +0.4.0 diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md new file mode 100644 index 0000000..448b28e --- /dev/null +++ b/docs/UPGRADING.md @@ -0,0 +1,97 @@ +# Upgrade gpu-perf-tune + +The full release history lives in [`CHANGELOG.md`](../CHANGELOG.md). + +## Moving from 0.3.x to 0.4.0 + +Refresh the client package or clone before reinstalling the MCP server. + +Codex clone: + +```bash +git pull --ff-only +make install-skills CLIENT=codex +``` + +Claude Code plugin: + +```bash +claude plugin update profile-and-optimize@profile-and-optimize-plugins +bash "$(ls -dt ~/.claude/plugins/cache/profile-and-optimize-plugins/profile-and-optimize/*/server/install.sh | head -1)" +claude mcp get plugin:profile-and-optimize:profile_and_optimize +``` + +The final command should report `Status: ✔ Connected`. + +The direct MCP installer and client configuration helper now require an +explicit client. Use one of the first-class paths from the repository root: + +```bash +make install-mcp CLIENT=codex +make install-mcp CLIENT=claude +``` + +When calling either helper directly, pass `--client codex` or +`--client claude`. An invocation without `--client` exits without changing a +client configuration. This replaces the old implicit Cursor target. + +Custom AIPerf command values must not contain `--api-key`. Pass the managed API +key option to the surrounding command, or set its documented environment +variable. The runtime supplies the key through the child process environment +or Kubernetes standard input and keeps it out of receipts and result payloads. + +Refresh the development environment before running the new language gates: + +```bash +bash plugins/profile-and-optimize/server/install.sh --with-dev +make all +``` + +## Moving from 0.2.x to 0.3.x + +### Codex + +Update the clone, refresh the linked skills, reinstall the MCP server, and +check the registered server: + +```bash +git pull --ff-only +make install-skills CLIENT=codex +make install-mcp CLIENT=codex +codex mcp get profile_and_optimize +``` + +The check should report `enabled: true`. Restart Codex before invoking a skill +or MCP tool. + +### Claude Code + +Update the plugin, reinstall the MCP server inside the current plugin cache +entry, and check the namespaced server: + +```bash +claude plugin update profile-and-optimize@profile-and-optimize-plugins +bash "$(ls -dt ~/.claude/plugins/cache/profile-and-optimize-plugins/profile-and-optimize/*/server/install.sh | head -1)" +claude mcp get plugin:profile-and-optimize:profile_and_optimize +``` + +The check should report `Status: ✔ Connected`. Restart Claude Code before +invoking a skill or MCP tool. + +### Required 0.3 command changes + +- AI tuning matrix, proposal, report, validation, and experiment creation + commands require an explicit `--space` file. +- MLPerf rules validation requires an explicit `--rules` file. +- Do not place acknowledgement flags in raw MCP `args`. Job submission uses + `i_understand_this_submits_jobs=true`. External publication uses + `i_understand_this_publishes_externally=true`. + +These checks fail closed when a required file or structured acknowledgement is +missing. Review the command before adding either acknowledgement. + +## Moving from 0.3.0 to 0.3.1 + +Version `0.3.1` adds security hardening without a new client configuration or +operator argument migration. Run the relevant client update steps above and +restart the client. diff --git a/plugins/profile-and-optimize/.claude-plugin/plugin.json b/plugins/profile-and-optimize/.claude-plugin/plugin.json index 8ce2b26..19f40f2 100644 --- a/plugins/profile-and-optimize/.claude-plugin/plugin.json +++ b/plugins/profile-and-optimize/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "profile-and-optimize", - "version": "0.3.1", + "version": "0.4.0", "description": "Claude Code adapter for gpu-perf-tune. Packages GPU inference Agent Skills, an opt-in provenance hook, and the bundled profile_and_optimize MCP server for benchmarking, profiling, optimization, and evidence-backed reports.", "author": { "name": "Project contributors" diff --git a/plugins/profile-and-optimize/README.md b/plugins/profile-and-optimize/README.md index c01a64a..1a7566d 100644 --- a/plugins/profile-and-optimize/README.md +++ b/plugins/profile-and-optimize/README.md @@ -1,6 +1,6 @@ # profile-and-optimize skills and Claude Code adapter -**Version v0.3.1** +**Version v0.4.0** This package contains 32 GPU inference profiling and optimization Agent Skills, the bundled `profile_and_optimize` MCP server, and the Claude Code marketplace diff --git a/plugins/profile-and-optimize/server/constraints-aa.txt b/plugins/profile-and-optimize/server/constraints-aa.txt index ca4e521..36bf444 100644 --- a/plugins/profile-and-optimize/server/constraints-aa.txt +++ b/plugins/profile-and-optimize/server/constraints-aa.txt @@ -10,9 +10,9 @@ # # Closure of tiktoken 0.13.0 as resolved on the validating venv (Python 3.14). tiktoken==0.13.0 -regex==2026.5.9 -charset-normalizer==3.4.7 +regex==2026.7.19 +charset-normalizer==3.5.0 requests==2.34.2 -certifi==2026.5.20 -idna==3.16 +certifi==2026.7.22 +idna==3.18 urllib3==2.7.0 diff --git a/plugins/profile-and-optimize/server/docs/README.md b/plugins/profile-and-optimize/server/docs/README.md index e4c9092..2cd1991 100644 --- a/plugins/profile-and-optimize/server/docs/README.md +++ b/plugins/profile-and-optimize/server/docs/README.md @@ -29,3 +29,4 @@ New local users should start with [`first-hour.md`](first-hour.md). Reviewers sh | Durable evidence families | [`../experiments/artifacts/`](../experiments/artifacts) | | Repo policy | [`../AGENTS.md`](../AGENTS.md) | | Review path | [`REVIEWERS.md`](../../../../REVIEWERS.md) | +| Version upgrades and migration notes | [`docs/UPGRADING.md`](../../../../docs/UPGRADING.md) | diff --git a/plugins/profile-and-optimize/server/findings/cli.py b/plugins/profile-and-optimize/server/findings/cli.py index c5201bc..5af0cd8 100644 --- a/plugins/profile-and-optimize/server/findings/cli.py +++ b/plugins/profile-and-optimize/server/findings/cli.py @@ -18,6 +18,7 @@ main, ) +__all__ = ["CONTRACT", "build_parser", "main"] if __name__ == "__main__": sys.exit(main()) diff --git a/plugins/profile-and-optimize/server/mcp_surface.py b/plugins/profile-and-optimize/server/mcp_surface.py index 0acf780..0cd627c 100644 --- a/plugins/profile-and-optimize/server/mcp_surface.py +++ b/plugins/profile-and-optimize/server/mcp_surface.py @@ -62,8 +62,10 @@ "mutates_cluster", } LIBRARIES = ( - "ai_tuning", "profile", - "perf_baseline", "evidence", + "ai_tuning", + "profile", + "perf_baseline", + "evidence", "slurm", "findings", # Backs the inference-perf-tune-report skill. @@ -143,9 +145,7 @@ def verify_canonical_counts() -> dict[str, int]: """ contract_tools = len(derive_tool_specs()) libraries = len(LIBRARIES) - assert libraries == _TOTAL_LIBRARIES, ( - f"LIBRARIES has {libraries} entries; expected {_TOTAL_LIBRARIES}" - ) + assert libraries == _TOTAL_LIBRARIES, f"LIBRARIES has {libraries} entries; expected {_TOTAL_LIBRARIES}" assert contract_tools == _TOTAL_CONTRACT_TOOLS, ( f"derive_tool_specs() returned {contract_tools} tools; " f"expected {_TOTAL_CONTRACT_TOOLS}. Update _TOTAL_CONTRACT_TOOLS " @@ -338,8 +338,7 @@ def _print_counts(*, json_mode: bool) -> int: "total_mcp_tools": _TOTAL_MCP_TOOLS, }, "live": live, - "verified": live["contract_tools"] == _TOTAL_CONTRACT_TOOLS - and live["libraries"] == _TOTAL_LIBRARIES, + "verified": live["contract_tools"] == _TOTAL_CONTRACT_TOOLS and live["libraries"] == _TOTAL_LIBRARIES, } print(json.dumps(payload, indent=2, sort_keys=True)) return 0 @@ -354,11 +353,16 @@ def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Contract-derived MCP-tool surface for the bundled profile_and_optimize MCP server.", ) - parser.add_argument("--json", action="store_true", help="emit machine-readable output for list / safety / counts / call") + parser.add_argument( + "--json", action="store_true", help="emit machine-readable output for list / safety / counts / call" + ) sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("list", help="list every derived MCP tool") sub.add_parser("safety", help="print derived tools grouped by safety class") - sub.add_parser("counts", help="print canonical counts (libraries / contract_tools / aux_tools / total_mcp_tools) and verify they match the live derivation") + sub.add_parser( + "counts", + help="print canonical counts (libraries / contract_tools / aux_tools / total_mcp_tools) and verify they match the live derivation", + ) call = sub.add_parser("call", help="invoke a derived MCP tool by name") call.add_argument("tool", help="tool name (for example, slurm_triage)") call.add_argument("args", nargs=argparse.REMAINDER, help="extra args forwarded to the underlying CLI verb") diff --git a/plugins/profile-and-optimize/server/pyproject.toml b/plugins/profile-and-optimize/server/pyproject.toml index 60c7c7c..6181448 100644 --- a/plugins/profile-and-optimize/server/pyproject.toml +++ b/plugins/profile-and-optimize/server/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "profile-and-optimize-server" -version = "0.3.1" +version = "0.4.0" description = "GPU inference profiling and optimization server. Ships 8 CLI libraries and 53 MCP tools total, including 51 contract-driven tools and 2 auxiliary tools." readme = "README.md" requires-python = ">=3.11" @@ -45,10 +45,11 @@ dev = [ "boto3>=1.34", "tiktoken>=0.7", "pre-commit>=4.0", - "pyright>=1.1", + "pyright>=1.1.411,<1.2", "pytest>=8.0,<10", "pytest-xdist>=3.6", - "ruff>=0.8", + "ruff>=0.16.3,<0.17", + "shellcheck-py==0.11.0.1", "skills-ref @ git+https://github.com/agentskills/agentskills.git@69ef37e9424c0a7ea9dd2293b559e43ec8176379#subdirectory=skills-ref", ] @@ -80,15 +81,42 @@ exclude = [ namespaces = true [tool.ruff] -target-version = "py312" +target-version = "py311" line-length = 120 extend-exclude = [ "experiments", "__pycache__", ".pytest_cache", - "tools/profile_and_optimize_mcp", ] +[tool.ruff.lint] +# Keep the enforced set explicit so a Ruff upgrade cannot silently change the +# repository gate. These rules cover syntax, imports, common correctness bugs, +# modern Python, exception handling, and Ruff-specific correctness checks. +select = [ + "E4", + "E7", + "E9", + "F", + "I", + "B", + "BLE", + "UP", + "RUF", + "S110", + "S112", +] + +[tool.ruff.lint.per-file-ignores] +# These entry points adjust sys.path before importing sibling command modules. +"**/tools/ai_tuning/*.py" = ["E402"] +"**/tools/perf_tune_report/importers/inference_drive_load.py" = ["E402"] +"**/tools/perf_tune_report/importers/zymtrace_kernels.py" = ["E402"] +# Several direct tests do the same to exercise the installed and source-tree +# entry points. Runtime behavior remains covered by pytest. +"**/test_*.py" = ["E402"] +"**/tests/*.py" = ["E402"] + [tool.pytest.ini_options] # The canonical `make pytest` target covers every checked-in offline Python # test under tools. Hardware checks must use a separate, explicitly named diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/ai_tuning.py b/plugins/profile-and-optimize/server/tools/ai_tuning/ai_tuning.py index fb5fac7..0a5e3cb 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/ai_tuning.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/ai_tuning.py @@ -24,6 +24,7 @@ # through the full tuner CLI. Re-exports preserve backwards compat for # callers that still import these names from this module. from tools.ai_tuning import helpers as _helpers +from tools.ai_tuning import safety as _safety from tools.ai_tuning.cmd_experiment import ( command_experiment_collect, command_experiment_create, @@ -45,8 +46,6 @@ from tools.ai_tuning.cmd_space import command_matrix, command_space from tools.ai_tuning.cmd_template_patch import command_template_patch_validate -from tools.ai_tuning import safety as _safety - EXPERIMENT_LEDGER_SCHEMA_VERSION = _safety.EXPERIMENT_LEDGER_SCHEMA_VERSION EXPERIMENT_STATUSES = _safety.EXPERIMENT_STATUSES FORBIDDEN_PATCH_PATTERNS = _safety.FORBIDDEN_PATCH_PATTERNS @@ -92,11 +91,20 @@ "safety": "writes_artifacts", "required": ("--space",), "optional": ( - "--raw-results-dir", "--raw-benchmark", "--min-runs", - "--objective", "--gb300-fabric-dir", "--gb300-fabric-require-clean", - "--gb300-node-selection-dir", "--gb300-fabric-localization-dir", - "--ledger", "--remaining-limit", "--template-hint-file", - "--template-hint-limit", "--error-limit", "--output", + "--raw-results-dir", + "--raw-benchmark", + "--min-runs", + "--objective", + "--gb300-fabric-dir", + "--gb300-fabric-require-clean", + "--gb300-node-selection-dir", + "--gb300-fabric-localization-dir", + "--ledger", + "--remaining-limit", + "--template-hint-file", + "--template-hint-limit", + "--error-limit", + "--output", ), "json": True, "ack": None, @@ -288,10 +296,7 @@ def build_parser() -> argparse.ArgumentParser: proposal_diff = proposal_subparsers.add_parser( "diff", - help=( - "Diff two proposal.json files by added, removed, or changed " - "experiment_id_prefix candidates." - ), + help=("Diff two proposal.json files by added, removed, or changed experiment_id_prefix candidates."), ) proposal_diff.add_argument("before", type=Path, help="earlier proposal.json") proposal_diff.add_argument("after", type=Path, help="later proposal.json") @@ -310,12 +315,8 @@ def build_parser() -> argparse.ArgumentParser: proposal_diff.set_defaults(func=command_proposal_diff) template_patch = subparsers.add_parser("template-patch", help="Template patch helpers") - template_patch_subparsers = template_patch.add_subparsers( - dest="template_patch_command", required=True - ) - patch_validate = template_patch_subparsers.add_parser( - "validate", help="Validate a context-anchored patch" - ) + template_patch_subparsers = template_patch.add_subparsers(dest="template_patch_command", required=True) + patch_validate = template_patch_subparsers.add_parser("validate", help="Validate a context-anchored patch") patch_validate.add_argument("patch", type=Path) patch_validate.add_argument("--apply", action="store_true") patch_validate.add_argument("--output-file", type=Path) @@ -385,9 +386,7 @@ def build_parser() -> argparse.ArgumentParser: experiment_submit.add_argument("--output", type=Path) experiment_submit.set_defaults(func=command_experiment_submit) - experiment_poll = experiment_subparsers.add_parser( - "poll", help="Poll read-only Slurm status and update the ledger" - ) + experiment_poll = experiment_subparsers.add_parser("poll", help="Poll read-only Slurm status and update the ledger") experiment_poll.add_argument("--ledger", type=Path, required=True) experiment_poll.add_argument("--status", action="append", default=["submitted", "running"]) experiment_poll.add_argument("--status-file", type=Path) @@ -441,9 +440,7 @@ def visit(node: argparse.ArgumentParser) -> None: for child in nested_action.choices.values(): visit(child) return - existing_options = { - opt for action in node._actions for opt in action.option_strings - } + existing_options = {opt for action in node._actions for opt in action.option_strings} if "--json" not in existing_options: node.add_argument( "--json", @@ -469,31 +466,39 @@ def __getattr__(name: str): return getattr(_helpers, name) raise AttributeError(name) -_EXPORT_DENYLIST = { - "Any", - "Path", - "annotations", - "argparse", - "contextlib", - "dt", - "gp_engine", - "hashlib", - "hyp_format", - "hyp_session", - "importlib", - "io", - "itertools", - "json", - "random", - "re", - "shutil", - "space_module", - "statistics", - "subprocess", - "sys", - "tpe_engine", -} -__all__ = [name for name in globals() if not name.startswith("_") and name not in _EXPORT_DENYLIST] + +__all__ = [ + "CONTRACT", + "EXAMPLE_SPACE", + "EXPERIMENT_LEDGER_SCHEMA_VERSION", + "EXPERIMENT_STATUSES", + "FORBIDDEN_PATCH_PATTERNS", + "MCP_DYNAMIC_ACKS", + "PROPOSAL_SCHEMA_VERSION", + "REPORT_SCHEMA_VERSION", + "REPO_ROOT", + "TEMPLATE_PATCH_SCHEMA_VERSION", + "build_parser", + "command_experiment_collect", + "command_experiment_create", + "command_experiment_poll", + "command_experiment_submit", + "command_experiment_summary", + "command_experiment_update", + "command_finalize", + "command_matrix", + "command_optimizer_compare", + "command_optimizer_history", + "command_optimizer_import_hyp", + "command_optimizer_propose", + "command_optimizer_status", + "command_proposal_diff", + "command_proposal_validate", + "command_report", + "command_space", + "command_template_patch_validate", + "main", +] if __name__ == "__main__": raise SystemExit(main()) diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/artifact_validation.py b/plugins/profile-and-optimize/server/tools/ai_tuning/artifact_validation.py index 2a7fe7d..73deab9 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/artifact_validation.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/artifact_validation.py @@ -67,9 +67,7 @@ def validate_result_file( benchmark_record = _last_record(records, "submission_benchmark") observed_benchmark = benchmark_record.get("value") if benchmark_record else None if observed_benchmark != benchmark: - errors.append( - f"{path}: submission_benchmark {observed_benchmark!r} does not match {benchmark!r}" - ) + errors.append(f"{path}: submission_benchmark {observed_benchmark!r} does not match {benchmark!r}") start = _last_record(records, "run_start") stop = _last_record(records, "run_stop") @@ -100,8 +98,7 @@ def validate_result_file( ( record for record in reversed(records) - if record.get("key") - in {"eval_accuracy", "eval_loss", "final_loss", "log_ppl"} + if record.get("key") in {"eval_accuracy", "eval_loss", "final_loss", "log_ppl"} ), None, ) @@ -142,9 +139,7 @@ def validate_raw_results_dir( return run_logs = sorted(raw_dir.glob("*_1.log")) if len(run_logs) < min_runs: - errors.append( - f"expected at least {min_runs} raw run logs in {raw_dir}, found {len(run_logs)}" - ) + errors.append(f"expected at least {min_runs} raw run logs in {raw_dir}, found {len(run_logs)}") if not list(raw_dir.glob("config_*.sh")): errors.append(f"missing config_*.sh in raw results directory: {raw_dir}") if not (raw_dir / "run.sub").is_file() and not list(raw_dir.glob("*.hyp")): @@ -159,9 +154,7 @@ def validate_raw_results_dir( validate_checker_output(errors, raw_dir / f"audit_{run_id}.out") if require_nccl_runtime: - runtime_text = "\n".join( - path.read_text(encoding="utf-8", errors="replace") for path in run_logs - ) + runtime_text = "\n".join(path.read_text(encoding="utf-8", errors="replace") for path in run_logs) if "NCCL" not in runtime_text: errors.append(f"raw results directory has no NCCL runtime evidence: {raw_dir}") @@ -179,9 +172,7 @@ def validate_fabric_evidence(errors: list[str], path: Path, require_clean: bool) _validate_evidence_dir(errors, path, "fabric evidence") if require_clean and path.is_dir(): text = "\n".join( - item.read_text(encoding="utf-8", errors="replace") - for item in path.rglob("*") - if item.is_file() + item.read_text(encoding="utf-8", errors="replace") for item in path.rglob("*") if item.is_file() ) if any(marker in text.upper() for marker in ("FAIL", "ERROR")): errors.append(f"fabric evidence contains failure markers: {path}") diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_experiment.py b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_experiment.py index 488f77b..9eb9af7 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_experiment.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_experiment.py @@ -15,8 +15,6 @@ from typing import Any REPO_ROOT = Path(__file__).resolve().parents[2] -from tools.shared.jsonutil import load_json - # Safety constants live in a small sibling module so reviewers can audit # the forbidden-pattern table and ledger-status enum without paging # through the full tuner CLI. Re-exports preserve backwards compat for @@ -48,6 +46,7 @@ EXPERIMENT_LEDGER_SCHEMA_VERSION, EXPERIMENT_STATUSES, ) +from tools.shared.jsonutil import load_json def command_experiment_create(args: argparse.Namespace) -> int: @@ -59,9 +58,7 @@ def command_experiment_create(args: argparse.Namespace) -> int: raise SystemExit("proposal must contain a non-empty candidates list") existing_ids = { - str(record.get("experiment_id")) - for record in read_ledger_records(args.ledger) - if record.get("experiment_id") + str(record.get("experiment_id")) for record in read_ledger_records(args.ledger) if record.get("experiment_id") } records = [] for index, candidate in enumerate(candidates): @@ -108,6 +105,7 @@ def command_experiment_create(args: argparse.Namespace) -> int: write_json(report, args.output) return 0 + def command_experiment_update(args: argparse.Namespace) -> int: if args.status not in EXPERIMENT_STATUSES: raise SystemExit(f"unknown status {args.status!r}; expected one of {sorted(EXPERIMENT_STATUSES)}") @@ -125,6 +123,7 @@ def command_experiment_update(args: argparse.Namespace) -> int: write_json(record, args.output) return 0 + def command_experiment_summary(args: argparse.Namespace) -> int: records = read_ledger_records(args.ledger) states = latest_experiment_states(records) @@ -153,14 +152,11 @@ def command_experiment_summary(args: argparse.Namespace) -> int: write_json(report, args.output) return 0 + def command_experiment_submit(args: argparse.Namespace) -> int: records = read_ledger_records(args.ledger) states = latest_experiment_states(records) - active = sum( - 1 - for record in states.values() - if record.get("status") in {"submitted", "running"} - ) + active = sum(1 for record in states.values() if record.get("status") in {"submitted", "running"}) capacity = max(args.max_concurrent - active, 0) statuses = set(args.status) candidates = select_submit_candidates(states, statuses, capacity, args.experiment_id) @@ -231,7 +227,11 @@ def command_experiment_submit(args: argparse.Namespace) -> int: "blocked", artifact_dir=record.get("artifact_dir"), notes="sbatch failed during gated submit", - extra={"submit_command": command, "wrapper": str(wrapper) if wrapper else None, "submit_result": result}, + extra={ + "submit_command": command, + "wrapper": str(wrapper) if wrapper else None, + "submit_result": result, + }, ) ) submissions.append(result) @@ -251,21 +251,17 @@ def command_experiment_submit(args: argparse.Namespace) -> int: write_json(report, args.output) return 0 + def command_experiment_poll(args: argparse.Namespace) -> int: records = read_ledger_records(args.ledger) states = latest_experiment_states(records) tracked = [ record for record in states.values() - if record.get("slurm_job_id") - and (not args.status or record.get("status") in set(args.status)) + if record.get("slurm_job_id") and (not args.status or record.get("status") in set(args.status)) ] job_ids = sorted({str(record["slurm_job_id"]) for record in tracked}) - slurm_statuses = ( - load_poll_statuses(args.status_file) - if args.status_file - else query_slurm_statuses(job_ids) - ) + slurm_statuses = load_poll_statuses(args.status_file) if args.status_file else query_slurm_statuses(job_ids) updates = [] observations = [] for record in tracked: @@ -273,9 +269,7 @@ def command_experiment_poll(args: argparse.Namespace) -> int: job_id = str(record["slurm_job_id"]) slurm = slurm_statuses.get(job_id) if slurm is None: - observations.append( - {"experiment_id": exp_id, "slurm_job_id": job_id, "found": False} - ) + observations.append({"experiment_id": exp_id, "slurm_job_id": job_id, "found": False}) continue new_status = str(slurm.get("status") or slurm_state_to_status(str(slurm.get("slurm_state", "")))) observation = { @@ -309,15 +303,14 @@ def command_experiment_poll(args: argparse.Namespace) -> int: write_json(report, args.output) return 0 + def command_experiment_collect(args: argparse.Namespace) -> int: records = read_ledger_records(args.ledger) states = latest_experiment_states(records) record = states.get(args.experiment_id) if record is None: raise SystemExit(f"unknown experiment id: {args.experiment_id}") - destination = args.destination or ( - Path(str(record["artifact_dir"])) if record.get("artifact_dir") else None - ) + destination = args.destination or (Path(str(record["artifact_dir"])) if record.get("artifact_dir") else None) if destination is None: raise SystemExit("collect requires --destination or an artifact_dir in the ledger") copy_artifacts(args.source, destination, args.overwrite) @@ -340,10 +333,7 @@ def command_experiment_collect(args: argparse.Namespace) -> int: "source": str(args.source), "destination": str(destination), "raw_benchmark": args.raw_benchmark, - "validation_errors": [ - {"code": error_code(message), "message": message} - for message in errors - ], + "validation_errors": [{"code": error_code(message), "message": message} for message in errors], "failure_classifications": failure_classifications, } }, @@ -355,10 +345,7 @@ def command_experiment_collect(args: argparse.Namespace) -> int: "artifact_dir": str(destination), "validation_passed": not errors, "failure_classifications": failure_classifications, - "validation_errors": [ - {"code": error_code(message), "message": message} - for message in errors - ], + "validation_errors": [{"code": error_code(message), "message": message} for message in errors], "update": update, } write_json(report, args.output) diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_optimizer.py b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_optimizer.py index a4f0ea5..327893d 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_optimizer.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_optimizer.py @@ -20,8 +20,6 @@ from tools.ai_tuning.optimizer import tpe as tpe_engine REPO_ROOT = Path(__file__).resolve().parents[2] -from tools.shared.jsonutil import load_json - # Safety constants live in a small sibling module so reviewers can audit # the forbidden-pattern table and ledger-status enum without paging # through the full tuner CLI. Re-exports preserve backwards compat for @@ -48,21 +46,18 @@ EXPERIMENT_LEDGER_SCHEMA_VERSION, PROPOSAL_SCHEMA_VERSION, ) +from tools.shared.jsonutil import load_json def command_optimizer_propose(args: argparse.Namespace) -> int: space = load_json(Path(args.space)) selected_objective = choose_objective(space, args.objective) objective = ( - str(selected_objective.get("name")) - if isinstance(selected_objective, dict) - else "mlperf_submission_readiness" + str(selected_objective.get("name")) if isinstance(selected_objective, dict) else "mlperf_submission_readiness" ) direction = _objective_direction(space, objective) domains = finite_parameter_domains(space) - selected_domains = [ - domain for domain in domains if not args.parameter or domain["name"] in set(args.parameter) - ] + selected_domains = [domain for domain in domains if not args.parameter or domain["name"] in set(args.parameter)] if not selected_domains: raise SystemExit("optimizer propose requires at least one finite parameter domain") selected_names = [domain["name"] for domain in selected_domains] @@ -84,9 +79,7 @@ def command_optimizer_propose(args: argparse.Namespace) -> int: truncated_default = False if strategy in {"deterministic_matrix", "random"}: - total_possible, parameter_iterator = iter_candidate_parameters( - selected_domains, strategy, rng - ) + total_possible, parameter_iterator = iter_candidate_parameters(selected_domains, strategy, rng) for parameters in parameter_iterator: if candidate_seen(parameters, states): continue @@ -109,7 +102,11 @@ def command_optimizer_propose(args: argparse.Namespace) -> int: except ValueError as exc: raise SystemExit(str(exc)) from exc observations = _observations_from_states(optimizer_space, states) - skip = {candidate_key({str(k): str(v) for k, v in record.get("parameters", {}).items()}) for record in states.values() if isinstance(record.get("parameters"), dict)} + skip = { + candidate_key({str(k): str(v) for k, v in record.get("parameters", {}).items()}) + for record in states.values() + if isinstance(record.get("parameters"), dict) + } engine_variant = variant or "tpe" if engine_variant == "tpe": decoded, engine_state = tpe_engine.propose( @@ -150,7 +147,11 @@ def command_optimizer_propose(args: argparse.Namespace) -> int: except ValueError as exc: raise SystemExit(str(exc)) from exc observations = _observations_from_states(optimizer_space, states) - skip = {candidate_key({str(k): str(v) for k, v in record.get("parameters", {}).items()}) for record in states.values() if isinstance(record.get("parameters"), dict)} + skip = { + candidate_key({str(k): str(v) for k, v in record.get("parameters", {}).items()}) + for record in states.values() + if isinstance(record.get("parameters"), dict) + } engine_variant = variant or "hyperband" config = _resolve_hyperband_config(space, engine_variant, args) state_in = None @@ -201,6 +202,7 @@ def command_optimizer_propose(args: argparse.Namespace) -> int: write_json(payload, args.output) return 0 + def command_optimizer_status(args: argparse.Namespace) -> int: space = load_json(Path(args.space)) objectives = objective_catalog(space) @@ -252,6 +254,7 @@ def command_optimizer_status(args: argparse.Namespace) -> int: write_json(payload, args.output) return 0 + def command_optimizer_history(args: argparse.Namespace) -> int: space = load_json(Path(args.space)) if args.space else {} records = read_ledger_records(args.ledger) @@ -307,6 +310,7 @@ def command_optimizer_history(args: argparse.Namespace) -> int: write_json(payload, args.output) return 0 + def command_optimizer_compare(args: argparse.Namespace) -> int: space = load_json(Path(args.space)) records = read_ledger_records(args.ledger) @@ -318,7 +322,9 @@ def command_optimizer_compare(args: argparse.Namespace) -> int: if record_b is None: raise SystemExit(f"unknown experiment id: {args.experiment_b}") - objective_name = args.objective or record_a.get("objective") or record_b.get("objective") or "mlperf_submission_readiness" + objective_name = ( + args.objective or record_a.get("objective") or record_b.get("objective") or "mlperf_submission_readiness" + ) direction = _objective_direction(space, objective_name) value_a = _experiment_state_value(record_a) @@ -335,9 +341,7 @@ def command_optimizer_compare(args: argparse.Namespace) -> int: else: winner = args.experiment_a if value_a < value_b else args.experiment_b - legality_changed = ( - record_a.get("config_patches") != record_b.get("config_patches") - ) + legality_changed = record_a.get("config_patches") != record_b.get("config_patches") payload = { "schema_version": 1, @@ -368,6 +372,7 @@ def command_optimizer_compare(args: argparse.Namespace) -> int: write_json(payload, args.output) return 0 + def command_optimizer_import_hyp(args: argparse.Namespace) -> int: template_path = Path(args.template) if not template_path.is_file(): diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_proposal.py b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_proposal.py index b8e51cb..b992c2f 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_proposal.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_proposal.py @@ -14,8 +14,6 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] -from tools.shared.jsonutil import load_json - # Safety constants live in a small sibling module so reviewers can audit # the forbidden-pattern table and ledger-status enum without paging # through the full tuner CLI. Re-exports preserve backwards compat for @@ -44,6 +42,7 @@ from tools.ai_tuning.safety import ( PROPOSAL_SCHEMA_VERSION, ) +from tools.shared.jsonutil import load_json def command_proposal_diff(args: argparse.Namespace) -> int: @@ -62,6 +61,7 @@ def command_proposal_diff(args: argparse.Namespace) -> int: args.output.write_text(rendered, encoding="utf-8") return 0 + def command_proposal_validate(args: argparse.Namespace) -> int: space = load_json(Path(args.space)) proposal = load_json(Path(args.proposal)) @@ -75,11 +75,7 @@ def command_proposal_validate(args: argparse.Namespace) -> int: schema_version = proposal.get("schema_version", 1) candidates = proposal.get("candidates", []) - if ( - schema_version != PROPOSAL_SCHEMA_VERSION - or not isinstance(candidates, list) - or not candidates - ): + if schema_version != PROPOSAL_SCHEMA_VERSION or not isinstance(candidates, list) or not candidates: report = { "schema_version": PROPOSAL_SCHEMA_VERSION, "valid_count": 0, @@ -89,9 +85,7 @@ def command_proposal_validate(args: argparse.Namespace) -> int: "index": 0, "valid": False, "error_codes": ["invalid_proposal_schema"], - "errors": [ - "proposal must use schema_version 1 and candidates must be a non-empty list" - ], + "errors": ["proposal must use schema_version 1 and candidates must be a non-empty list"], "warning_codes": [], "warnings": [], } @@ -226,9 +220,7 @@ def command_proposal_validate(args: argparse.Namespace) -> int: result["valid"] = False result["error_codes"].append("missing_config_patch") result["errors"].append(f"{name}: config mutation requires a config_patches entry") - elif required["method"] == "template_patch" and not ( - patch.get("patch_file") or patch.get("changes") - ): + elif required["method"] == "template_patch" and not (patch.get("patch_file") or patch.get("changes")): result["valid"] = False result["error_codes"].append("invalid_config_patch_contract") result["errors"].append(f"{name}: template_patch contract requires patch_file or inline changes") diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_report.py b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_report.py index e373547..92b0fba 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_report.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_report.py @@ -13,8 +13,6 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] -from tools.shared.jsonutil import load_json - # Safety constants live in a small sibling module so reviewers can audit # the forbidden-pattern table and ledger-status enum without paging # through the full tuner CLI. Re-exports preserve backwards compat for @@ -36,6 +34,7 @@ from tools.ai_tuning.safety import ( REPORT_SCHEMA_VERSION, ) +from tools.shared.jsonutil import load_json def command_report(args: argparse.Namespace) -> int: @@ -46,15 +45,9 @@ def command_report(args: argparse.Namespace) -> int: validator_args = argparse.Namespace(**vars(args)) validator_args.min_runs = required_runs validation_errors = run_validator(validator_args) - raw_summaries = [ - summarize_raw_dir(Path(raw_dir).resolve(), args.raw_benchmark) - for raw_dir in args.raw_results_dir - ] + raw_summaries = [summarize_raw_dir(Path(raw_dir).resolve(), args.raw_benchmark) for raw_dir in args.raw_results_dir] successful_runs = sum( - 1 - for raw in raw_summaries - for run in raw["runs"] - if run.get("status") == "success" and not run.get("errors") + 1 for raw in raw_summaries for run in raw["runs"] if run.get("status") == "success" and not run.get("errors") ) quality_target = effective_quality_target(target, selected_objective) report = { @@ -73,8 +66,7 @@ def command_report(args: argparse.Namespace) -> int: "passed": not validation_errors, "error_count": len(validation_errors), "errors": [ - {"code": error_code(message), "message": message} - for message in validation_errors[: args.error_limit] + {"code": error_code(message), "message": message} for message in validation_errors[: args.error_limit] ], "truncated": len(validation_errors) > args.error_limit, }, @@ -95,9 +87,7 @@ def command_report(args: argparse.Namespace) -> int: validation_errors, ), "raw_results": raw_summaries, - "gb300_fabric": summarize_optional_dirs( - args.gb300_fabric_dir, ["evidence.json", "run-context.json"] - ), + "gb300_fabric": summarize_optional_dirs(args.gb300_fabric_dir, ["evidence.json", "run-context.json"]), "gb300_node_selection": summarize_optional_dirs( args.gb300_node_selection_dir, ["selector-config.json", "node-scores.json"] ), diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_space.py b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_space.py index 9f3660c..9a53eca 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_space.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_space.py @@ -15,8 +15,6 @@ from typing import Any REPO_ROOT = Path(__file__).resolve().parents[2] -from tools.shared.jsonutil import load_json - # Safety constants live in a small sibling module so reviewers can audit # the forbidden-pattern table and ledger-status enum without paging # through the full tuner CLI. Re-exports preserve backwards compat for @@ -25,6 +23,7 @@ from tools.ai_tuning.safety import ( PROPOSAL_SCHEMA_VERSION, ) +from tools.shared.jsonutil import load_json def command_space(args: argparse.Namespace) -> int: @@ -50,6 +49,7 @@ def command_space(args: argparse.Namespace) -> int: write_json(payload, args.output) return 0 + def command_matrix(args: argparse.Namespace) -> int: space = load_json(Path(args.space)) selected_objective = choose_objective(space) @@ -73,10 +73,7 @@ def command_matrix(args: argparse.Namespace) -> int: for values in itertools.product(*(item["values"] for item in selected)): if len(candidates) >= args.limit: break - parameters = { - selected[index]["name"]: values[index] - for index in range(len(selected)) - } + parameters = {selected[index]["name"]: values[index] for index in range(len(selected))} candidates.append( { "parameters": parameters, diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_template_patch.py b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_template_patch.py index 360fb43..7e1c182 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_template_patch.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/cmd_template_patch.py @@ -13,14 +13,12 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] -from tools.shared.jsonutil import load_json - # Safety constants live in a small sibling module so reviewers can audit # the forbidden-pattern table and ledger-status enum without paging # through the full tuner CLI. Re-exports preserve backwards compat for # callers that still import these names from this module. - from tools.ai_tuning.helpers import default_derived_path, evaluate_template_patch_request, write_json +from tools.shared.jsonutil import load_json def command_template_patch_validate(args: argparse.Namespace) -> int: diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/helpers.py b/plugins/profile-and-optimize/server/tools/ai_tuning/helpers.py index 5e4c2fc..f4ed14a 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/helpers.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/helpers.py @@ -23,8 +23,6 @@ from tools.ai_tuning.optimizer import tpe as tpe_engine REPO_ROOT = Path(__file__).resolve().parents[2] -from tools.shared.jsonutil import load_json - # Safety constants live in a small sibling module so reviewers can audit # the forbidden-pattern table and ledger-status enum without paging # through the full tuner CLI. Re-exports preserve backwards compat for @@ -35,6 +33,7 @@ FORBIDDEN_PATCH_PATTERNS, TEMPLATE_PATCH_SCHEMA_VERSION, ) +from tools.shared.jsonutil import load_json def load_validate_artifacts() -> Any: @@ -42,6 +41,7 @@ def load_validate_artifacts() -> Any: return artifact_validation + def write_json(payload: Any, output: Path | None) -> None: text = json.dumps(payload, indent=2, sort_keys=True) + "\n" if output is None: @@ -50,21 +50,26 @@ def write_json(payload: Any, output: Path | None) -> None: output.parent.mkdir(parents=True, exist_ok=True) output.write_text(text, encoding="utf-8") + def append_jsonl(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(payload, sort_keys=True) + "\n") + def utc_timestamp() -> str: return dt.datetime.now(dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + def stable_hash(payload: Any) -> str: encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(encoded).hexdigest() + def short_hash(payload: Any) -> str: return stable_hash(payload)[:12] + def error_code(message: str) -> str: lowered = message.lower() if "missing raw results directory" in lowered: @@ -89,6 +94,7 @@ def error_code(message: str) -> str: return "missing_jsonl" return re.sub(r"[^a-z0-9]+", "_", lowered).strip("_")[:80] or "validation_error" + def parameter_index(space: dict[str, Any]) -> dict[str, dict[str, Any]]: return { str(param["name"]): param @@ -96,6 +102,7 @@ def parameter_index(space: dict[str, Any]) -> dict[str, dict[str, Any]]: if isinstance(param, dict) and "name" in param } + def objective_index(space: dict[str, Any]) -> dict[str, dict[str, Any]]: return { str(objective["name"]): objective @@ -103,9 +110,11 @@ def objective_index(space: dict[str, Any]) -> dict[str, dict[str, Any]]: if isinstance(objective, dict) and "name" in objective } + def objective_catalog(space: dict[str, Any]) -> list[dict[str, Any]]: return [objective for objective in space.get("objectives", []) if isinstance(objective, dict)] + def choose_objective(space: dict[str, Any], requested: str | None = None) -> dict[str, Any] | None: objectives = objective_catalog(space) if not objectives: @@ -117,6 +126,7 @@ def choose_objective(space: dict[str, Any], requested: str | None = None) -> dic raise SystemExit(f"unknown objective {requested!r}") return selected + def config_contract_index(space: dict[str, Any]) -> dict[str, dict[str, Any]]: return { str(contract["parameter"]): contract @@ -124,6 +134,7 @@ def config_contract_index(space: dict[str, Any]) -> dict[str, dict[str, Any]]: if isinstance(contract, dict) and "parameter" in contract } + def normalize_config_patches(candidate: dict[str, Any]) -> dict[str, dict[str, Any]]: patches = candidate.get("config_patches", []) if not isinstance(patches, list): @@ -134,6 +145,7 @@ def normalize_config_patches(candidate: dict[str, Any]) -> dict[str, dict[str, A normalized[str(patch["parameter"])] = patch return normalized + def normalize_candidate_parameters(candidate: dict[str, Any]) -> dict[str, str]: raw = candidate.get("parameters", {}) if isinstance(raw, dict): @@ -148,6 +160,7 @@ def normalize_candidate_parameters(candidate: dict[str, Any]) -> dict[str, str]: return normalized return {} + def value_allowed(param: dict[str, Any], value: str) -> tuple[bool, str | None]: kind = param.get("kind", "string") if "values" in param: @@ -177,6 +190,7 @@ def value_allowed(param: dict[str, Any], value: str) -> tuple[bool, str | None]: return False, f"value {value!r} is not a boolean" return True, None + def run_validator(args: argparse.Namespace) -> list[str]: validator = load_validate_artifacts() errors: list[str] = [] @@ -191,24 +205,21 @@ def run_validator(args: argparse.Namespace) -> list[str]: require_nccl_runtime=require_nccl_runtime, ) for fabric_dir in args.gb300_fabric_dir: - validator.validate_fabric_evidence( - errors, Path(fabric_dir).resolve(), args.gb300_fabric_require_clean - ) + validator.validate_fabric_evidence(errors, Path(fabric_dir).resolve(), args.gb300_fabric_require_clean) for selection_dir in args.gb300_node_selection_dir: validator.validate_node_selection_dir(errors, Path(selection_dir).resolve()) for localization_dir in args.gb300_fabric_localization_dir: validator.validate_fabric_localization_dir(errors, Path(localization_dir).resolve()) return errors + def summarize_raw_dir(raw_dir: Path, benchmark: str) -> dict[str, Any]: validator = load_validate_artifacts() run_summaries = [] for run_log in sorted(raw_dir.glob("*_1.log")): per_run_errors: list[str] = [] with contextlib.redirect_stdout(io.StringIO()): - status, elapsed_minutes, final_metric = validator.validate_result_file( - per_run_errors, benchmark, run_log - ) + status, elapsed_minutes, final_metric = validator.validate_result_file(per_run_errors, benchmark, run_log) run_id = validator.run_id_from_log_path(run_log) compliance_path = raw_dir / f"compliance_{run_id}.out" audit_path = raw_dir / f"audit_{run_id}.out" @@ -221,9 +232,7 @@ def summarize_raw_dir(raw_dir: Path, benchmark: str) -> dict[str, Any]: "final_log_ppl": final_metric, "compliance_output": str(compliance_path), "audit_output": str(audit_path), - "errors": [ - {"code": error_code(message), "message": message} for message in per_run_errors - ], + "errors": [{"code": error_code(message), "message": message} for message in per_run_errors], } ) return { @@ -232,17 +241,19 @@ def summarize_raw_dir(raw_dir: Path, benchmark: str) -> dict[str, Any]: "run_log_count": len(run_summaries), "config_count": len(list(raw_dir.glob("config_*.sh"))), "launcher_count": len(list(raw_dir.glob("*.hyp"))) + int((raw_dir / "run.sub").is_file()), - "env_log_count": len(list(raw_dir.glob("container-env-*.log"))) - + len(list(raw_dir.glob("*_env.log"))), + "env_log_count": len(list(raw_dir.glob("container-env-*.log"))) + len(list(raw_dir.glob("*_env.log"))), "runs": run_summaries, } + def mean_or_none(values: list[float]) -> float | None: return statistics.fmean(values) if values else None + def stdev_or_none(values: list[float]) -> float | None: return statistics.stdev(values) if len(values) >= 2 else None + def effective_quality_target(target: dict[str, Any], objective: dict[str, Any] | None) -> float | int | None: if objective and isinstance(objective.get("quality_target"), int | float): return objective["quality_target"] @@ -250,6 +261,7 @@ def effective_quality_target(target: dict[str, Any], objective: dict[str, Any] | return target["quality_target"] return None + def effective_required_runs( raw_benchmark: str, min_runs: int | None, @@ -263,6 +275,7 @@ def effective_required_runs( return 5 return 1 + def build_trial_analysis( raw_summaries: list[dict[str, Any]], target: dict[str, Any], @@ -273,15 +286,9 @@ def build_trial_analysis( ) -> dict[str, Any]: runs = [run for raw in raw_summaries for run in raw.get("runs", [])] elapsed_values = [ - float(run["elapsed_minutes"]) - for run in runs - if isinstance(run.get("elapsed_minutes"), int | float) - ] - metric_values = [ - float(run["final_log_ppl"]) - for run in runs - if isinstance(run.get("final_log_ppl"), int | float) + float(run["elapsed_minutes"]) for run in runs if isinstance(run.get("elapsed_minutes"), int | float) ] + metric_values = [float(run["final_log_ppl"]) for run in runs if isinstance(run.get("final_log_ppl"), int | float)] quality_target = effective_quality_target(target, objective) if isinstance(quality_target, int | float): quality_pass_count = sum(value <= float(quality_target) for value in metric_values) @@ -307,12 +314,8 @@ def build_trial_analysis( "primary_objective": objective_name, "primary_metric": objective.get("primary_metric") if isinstance(objective, dict) else "submission_ready", "direction": objective.get("direction") if isinstance(objective, dict) else "maximize", - "objective_progress_ratio": min(successful_runs / required_runs, 1.0) - if required_runs - else None, - "submission_readiness_score": min(successful_runs / required_runs, 1.0) - if required_runs - else None, + "objective_progress_ratio": min(successful_runs / required_runs, 1.0) if required_runs else None, + "submission_readiness_score": min(successful_runs / required_runs, 1.0) if required_runs else None, "mean_elapsed_minutes": mean_or_none(elapsed_values), "mean_final_log_ppl": mean_or_none(metric_values), "quality_target": quality_target, @@ -352,6 +355,7 @@ def build_trial_analysis( }, } + def build_objective_catalog(space: dict[str, Any]) -> list[dict[str, Any]]: objectives = objective_catalog(space) if objectives: @@ -368,6 +372,7 @@ def build_objective_catalog(space: dict[str, Any]) -> list[dict[str, Any]]: } ] + def run_id_from_log(path: Path) -> str: stem = path.stem if stem.endswith("_1"): @@ -376,6 +381,7 @@ def run_id_from_log(path: Path) -> str: return stem[:-3] return stem + def discover_finalize_logs(log_dir: Path, run_ids: list[str]) -> list[Path]: if run_ids: logs = [log_dir / f"{run_id}_1.log" for run_id in run_ids] @@ -386,6 +392,7 @@ def discover_finalize_logs(log_dir: Path, run_ids: list[str]) -> list[Path]: raise SystemExit(f"missing run log(s): {', '.join(missing)}") return logs + def validate_finalize_run(log_dir: Path, log_path: Path, benchmark: str) -> dict[str, Any]: validator = load_validate_artifacts() errors: list[str] = [] @@ -408,6 +415,7 @@ def validate_finalize_run(log_dir: Path, log_path: Path, benchmark: str) -> dict "errors": [{"code": error_code(message), "message": message} for message in errors], } + def finalize_copy_candidates( log_dir: Path, workdir: Path, @@ -450,6 +458,7 @@ def finalize_copy_candidates( deduped.append((src, dst)) return deduped + def summarize_json_file(path: Path) -> Any: if not path.is_file(): return None @@ -458,6 +467,7 @@ def summarize_json_file(path: Path) -> Any: except json.JSONDecodeError as exc: return {"error": f"malformed JSON: {exc}"} + def summarize_optional_dirs(paths: list[str], filenames: list[str]) -> list[dict[str, Any]]: summaries = [] for raw_path in paths: @@ -468,6 +478,7 @@ def summarize_optional_dirs(paths: list[str], filenames: list[str]) -> list[dict summaries.append(summary) return summaries + def finite_parameter_domains(space: dict[str, Any]) -> list[dict[str, Any]]: domains = [] for param in space.get("parameters", []): @@ -490,9 +501,8 @@ def finite_parameter_domains(space: dict[str, Any]) -> list[dict[str, Any]]: domains.sort(key=lambda item: item["name"]) return domains -def build_parameter_coverage( - domains: list[dict[str, Any]], states: dict[str, dict[str, Any]] -) -> list[dict[str, Any]]: + +def build_parameter_coverage(domains: list[dict[str, Any]], states: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: coverage = [] for domain in domains: observed_counts = {value: 0 for value in domain["values"]} @@ -517,6 +527,7 @@ def build_parameter_coverage( ) return coverage + def build_remaining_candidates( domains: list[dict[str, Any]], states: dict[str, dict[str, Any]], @@ -539,10 +550,7 @@ def build_remaining_candidates( for values in itertools.product(*(domain["values"] for domain in domains)): if len(candidates) >= limit: break - parameters = { - domains[index]["name"]: values[index] - for index in range(len(domains)) - } + parameters = {domains[index]["name"]: values[index] for index in range(len(domains))} key = candidate_key(parameters) if key in tried_keys: continue @@ -560,6 +568,7 @@ def build_remaining_candidates( "candidates": candidates, } + def build_circuit_breakers( counts: dict[str, int], coverage: list[dict[str, Any]], remaining: dict[str, Any] ) -> list[dict[str, str]]: @@ -605,6 +614,7 @@ def build_circuit_breakers( ) return breakers + def split_name_value(value: str) -> tuple[str | None, str | None]: if "=" in value: name, parsed = value.split("=", 1) @@ -614,6 +624,7 @@ def split_name_value(value: str) -> tuple[str | None, str | None]: return parts[0].strip() or None, parts[1].strip().strip('"') or None return value.strip() or None, None + def parse_template_hint_line(line: str, line_number: int) -> dict[str, Any] | None: trimmed = line.strip() if not trimmed or "hypertune." in trimmed: @@ -638,6 +649,7 @@ def parse_template_hint_line(line: str, line_number: int) -> dict[str, Any] | No return {"kind": "argument", "name": name, "value": value, "line_number": line_number, "raw": line} return None + def build_template_hints(paths: list[str], limit: int) -> dict[str, Any]: if limit < 0: raise SystemExit("--template-hint-limit must be non-negative") @@ -668,6 +680,7 @@ def build_template_hints(paths: list[str], limit: int) -> dict[str, Any]: "hints": hints[:limit], } + def build_agent_session_report( space: dict[str, Any], ledger: Path | None, @@ -731,6 +744,7 @@ def build_agent_session_report( }, } + def candidate_seen(parameters: dict[str, str], states: dict[str, dict[str, Any]]) -> bool: key = candidate_key(parameters) for record in states.values(): @@ -740,6 +754,7 @@ def candidate_seen(parameters: dict[str, str], states: dict[str, dict[str, Any]] return True return False + def decode_combination(index: int, domains: list[dict[str, Any]]) -> dict[str, str]: values: list[str] = [] remaining = index @@ -748,10 +763,8 @@ def decode_combination(index: int, domains: list[dict[str, Any]]) -> dict[str, s remaining, offset = divmod(remaining, len(domain_values)) values.append(domain_values[offset]) values.reverse() - return { - domains[position]["name"]: values[position] - for position in range(len(domains)) - } + return {domains[position]["name"]: values[position] for position in range(len(domains))} + def iter_candidate_parameters( domains: list[dict[str, Any]], @@ -762,6 +775,7 @@ def iter_candidate_parameters( for domain in domains: total_possible *= len(domain["values"]) if strategy == "random": + def iterator() -> Any: seen_indices: set[int] = set() while len(seen_indices) < total_possible: @@ -775,19 +789,18 @@ def iterator() -> Any: def iterator() -> Any: for values in itertools.product(*(domain["values"] for domain in domains)): - yield { - domains[index]["name"]: values[index] - for index in range(len(domains)) - } + yield {domains[index]["name"]: values[index] for index in range(len(domains))} return total_possible, iterator() + def _build_optimizer_space( space: dict[str, Any], selected_names: list[str] | None, ) -> space_module.Space: return space_module.Space.from_manifest(space, parameter_names=selected_names) + def _observations_from_states( optimizer_space: space_module.Space, states: dict[str, dict[str, Any]], @@ -811,6 +824,7 @@ def _observations_from_states( observations.append(tpe_engine.Observation(vector=vector, value=value)) return observations + def _objective_direction(space: dict[str, Any], objective_name: str) -> str: for objective in space.get("objectives", []): if isinstance(objective, dict) and str(objective.get("name")) == objective_name: @@ -819,14 +833,22 @@ def _objective_direction(space: dict[str, Any], objective_name: str) -> str: return direction return "maximize" -def _resolve_hyperband_config(space: dict[str, Any], variant: str, args: argparse.Namespace) -> hb_engine.HyperbandConfig: - optimizer_block = space.get("optimizer", {}) if isinstance(space.get("optimizer"), dict) else {} - block = optimizer_block.get(variant) if isinstance(optimizer_block.get(variant), dict) else {} + +def _resolve_hyperband_config( + space: dict[str, Any], variant: str, args: argparse.Namespace +) -> hb_engine.HyperbandConfig: + raw_optimizer_block = space.get("optimizer") + optimizer_block = raw_optimizer_block if isinstance(raw_optimizer_block, dict) else {} + raw_variant_block = optimizer_block.get(variant) + block = raw_variant_block if isinstance(raw_variant_block, dict) else {} eta = int(args.eta) if args.eta is not None else int(block.get("eta", 3)) min_budget = float(args.min_budget) if args.min_budget is not None else float(block.get("min_budget", 1.0)) - max_budget = float(args.max_budget) if args.max_budget is not None else float(block.get("max_budget", min_budget * (eta ** 3))) + max_budget = ( + float(args.max_budget) if args.max_budget is not None else float(block.get("max_budget", min_budget * (eta**3))) + ) return hb_engine.HyperbandConfig(eta=eta, min_budget=min_budget, max_budget=max_budget) + def _experiment_state_value(record: dict[str, Any]) -> float | None: raw = record.get("result_value") if raw is None: @@ -836,6 +858,7 @@ def _experiment_state_value(record: dict[str, Any]) -> float | None: except (TypeError, ValueError): return None + def load_history_keys(paths: list[str]) -> set[str]: keys: set[str] = set() for raw_path in paths: @@ -854,6 +877,7 @@ def load_history_keys(paths: list[str]) -> set[str]: keys.add(candidate_key({str(k): str(v) for k, v in parameters.items()})) return keys + def load_ledger_keys(path: Path | None) -> set[str]: if path is None: return set() @@ -864,12 +888,15 @@ def load_ledger_keys(path: Path | None) -> set[str]: keys.add(candidate_key({str(k): str(v) for k, v in parameters.items()})) return keys + def candidate_key(parameters: dict[str, str]) -> str: return "|".join(f"{name}={parameters[name]}" for name in sorted(parameters)) + def experiment_id(space_id: str, index: int, parameters: dict[str, str]) -> str: return f"exp-{short_hash({'space': space_id, 'index': index, 'parameters': parameters})}" + def normalize_priority(candidate: dict[str, Any], default: int) -> int: raw_priority = candidate.get("priority", default) if isinstance(raw_priority, bool): @@ -882,6 +909,7 @@ def normalize_priority(candidate: dict[str, Any], default: int) -> int: raise SystemExit("candidate priority must be at least 1") return priority + def infer_experiment_shape(parameters: dict[str, str], space: dict[str, Any]) -> dict[str, Any]: target = space.get("target", {}) nodes = ( @@ -898,6 +926,7 @@ def infer_experiment_shape(parameters: dict[str, str], space: dict[str, Any]) -> "benchmark": target.get("benchmark"), } + def read_ledger_records(path: Path) -> list[dict[str, Any]]: if not path.is_file(): return [] @@ -914,6 +943,7 @@ def read_ledger_records(path: Path) -> list[dict[str, Any]]: records.append(record) return records + def latest_experiment_states(records: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: states: dict[str, dict[str, Any]] = {} for record in records: @@ -929,6 +959,7 @@ def latest_experiment_states(records: list[dict[str, Any]]) -> dict[str, dict[st current[key] = value return states + def append_experiment_update( ledger: Path, experiment_id_value: str, @@ -956,6 +987,7 @@ def append_experiment_update( append_jsonl(ledger, record) return record + def planned_submit_command(record: dict[str, Any], script: Path, extra_args: list[str]) -> list[str]: shape = record.get("shape", {}) if isinstance(record.get("shape"), dict) else {} command = ["sbatch", "--parsable"] @@ -969,9 +1001,11 @@ def planned_submit_command(record: dict[str, Any], script: Path, extra_args: lis command.append(str(script)) return command + def shell_quote(value: str) -> str: return "'" + value.replace("'", "'\"'\"'") + "'" + def extract_sbatch_directives(script: Path) -> list[str]: directives = [] for line in script.read_text(encoding="utf-8", errors="replace").splitlines(): @@ -980,11 +1014,13 @@ def extract_sbatch_directives(script: Path) -> list[str]: directives.append(line) return directives + def patch_target_name(raw_path: str | None) -> str | None: if not raw_path: return None return Path(str(raw_path)).name + def load_patch_request_from_descriptor(patch: dict[str, Any], script: Path) -> dict[str, Any]: patch_file = patch.get("patch_file") if patch_file: @@ -1009,6 +1045,7 @@ def load_patch_request_from_descriptor(patch: dict[str, Any], script: Path) -> d request["target_file"] = patch.get("target_file") return request + def materialize_record_config_patches( record: dict[str, Any], script: Path, @@ -1052,6 +1089,7 @@ def materialize_record_config_patches( source_lines.append(f"source {shell_quote(str(output))}") return source_lines, target_script_override, derived_files + def materialize_submit_wrapper( record: dict[str, Any], script: Path, @@ -1116,14 +1154,15 @@ def materialize_submit_wrapper( if local_script.exists() and local_script.read_bytes() != script.read_bytes() and not overwrite: raise SystemExit(f"wrapper script copy already exists with different contents: {local_script}") shutil.copy2(script, local_script) - target_script = f"$(cd -- \"$(dirname -- \"$0\")\" && pwd)/{script.name}" + target_script = f'$(cd -- "$(dirname -- "$0")" && pwd)/{script.name}' for derived_path in derived_patch_files: lines.append(f"# derived_patch={derived_path}") - lines.append(f"exec bash {shell_quote(target_script)} \"$@\"") + lines.append(f'exec bash {shell_quote(target_script)} "$@"') path.write_text("\n".join(lines) + "\n", encoding="utf-8") path.chmod(0o755) return path + def select_submit_candidates( states: dict[str, dict[str, Any]], statuses: set[str], @@ -1137,11 +1176,7 @@ def select_submit_candidates( if missing: raise SystemExit(f"unknown experiment id(s): {', '.join(missing)}") return [states[exp_id] for exp_id in experiment_ids] - candidates = [ - record - for record in states.values() - if str(record.get("status", "")) in statuses - ] + candidates = [record for record in states.values() if str(record.get("status", "")) in statuses] candidates.sort( key=lambda record: ( int(record.get("priority", 1_000_000)), @@ -1150,6 +1185,7 @@ def select_submit_candidates( ) return candidates[:capacity] + def slurm_state_to_status(state: str) -> str: normalized = state.upper().split()[0] if normalized in {"PENDING", "CONFIGURING", "COMPLETING"}: @@ -1164,6 +1200,7 @@ def slurm_state_to_status(state: str) -> str: return "failed" return "blocked" + def load_poll_statuses(path: Path) -> dict[str, dict[str, Any]]: payload = load_json(path) if isinstance(payload, dict) and isinstance(payload.get("jobs"), list): @@ -1179,6 +1216,7 @@ def load_poll_statuses(path: Path) -> dict[str, dict[str, Any]]: statuses[str(row["slurm_job_id"])] = row return statuses + def query_slurm_statuses(job_ids: list[str]) -> dict[str, dict[str, Any]]: if not job_ids: return {} @@ -1210,6 +1248,7 @@ def query_slurm_statuses(job_ids: list[str]) -> dict[str, dict[str, Any]]: } return statuses + def copy_artifacts(source: Path, destination: Path, overwrite: bool) -> None: if not source.is_dir(): raise SystemExit(f"missing artifact source directory: {source}") @@ -1225,6 +1264,7 @@ def copy_artifacts(source: Path, destination: Path, overwrite: bool) -> None: raise SystemExit(f"artifact destination file exists: {target}") shutil.copy2(child, target) + def validate_collected_artifacts(destination: Path, benchmark: str, min_runs: int) -> list[str]: validator = load_validate_artifacts() errors: list[str] = [] @@ -1242,6 +1282,7 @@ def validate_collected_artifacts(destination: Path, benchmark: str, min_runs: in ) return errors + FAILURE_PATTERNS: tuple[tuple[str, str, str], ...] = ( ("failure", "nccl_ib_create_ah_no_device", "ibv_create_ah failed with error No such device"), ("failure", "nccl_ib_modify_qp_no_device", "ibv_modify_qp failed with 19 No such device"), @@ -1259,6 +1300,7 @@ def validate_collected_artifacts(destination: Path, benchmark: str, min_runs: in ("failure", "slurm_node_fail", "NODE_FAIL"), ) + def classify_failure_text(text: str) -> list[dict[str, Any]]: findings = [] for severity, code, pattern in FAILURE_PATTERNS: @@ -1267,16 +1309,13 @@ def classify_failure_text(text: str) -> list[dict[str, Any]]: findings.append({"severity": severity, "code": code, "pattern": pattern, "count": count}) return findings + def classify_artifacts(path: Path) -> list[dict[str, Any]]: findings: list[dict[str, Any]] = [] if path.is_file(): paths = [path] elif path.is_dir(): - paths = sorted( - item - for item in path.rglob("*") - if item.is_file() and item.suffix in {".out", ".log", ".txt"} - ) + paths = sorted(item for item in path.rglob("*") if item.is_file() and item.suffix in {".out", ".log", ".txt"}) else: return findings for item in paths: @@ -1287,6 +1326,7 @@ def classify_artifacts(path: Path) -> list[dict[str, Any]]: findings.append(finding) return findings + def _proposal_candidate_index(proposal: dict[str, Any]) -> dict[str, dict[str, Any]]: """Index candidates by experiment_id_prefix for diffing.""" out: dict[str, dict[str, Any]] = {} @@ -1297,15 +1337,13 @@ def _proposal_candidate_index(proposal: dict[str, Any]) -> dict[str, dict[str, A if not isinstance(candidate, dict): continue key = str( - candidate.get("experiment_id_prefix") - or candidate.get("experiment_id") - or candidate.get("id") - or "" + candidate.get("experiment_id_prefix") or candidate.get("experiment_id") or candidate.get("id") or "" ).strip() if key: out[key] = candidate return out + def _proposal_diff_payload( *, before_path: Path, @@ -1316,12 +1354,8 @@ def _proposal_diff_payload( before = load_json(before_path) after = load_json(after_path) - before_hash = hashlib.sha256( - json.dumps(before, sort_keys=True).encode("utf-8") - ).hexdigest() - after_hash = hashlib.sha256( - json.dumps(after, sort_keys=True).encode("utf-8") - ).hexdigest() + before_hash = hashlib.sha256(json.dumps(before, sort_keys=True).encode("utf-8")).hexdigest() + after_hash = hashlib.sha256(json.dumps(after, sort_keys=True).encode("utf-8")).hexdigest() if before_hash == after_hash: return { "schema_version": 1, @@ -1347,20 +1381,13 @@ def _proposal_diff_payload( continue added_params = sorted(set(a_params) - set(b_params)) removed_params = sorted(set(b_params) - set(a_params)) - changed_params = sorted( - k - for k in set(a_params) & set(b_params) - if str(a_params[k]) != str(b_params[k]) - ) + changed_params = sorted(k for k in set(a_params) & set(b_params) if str(a_params[k]) != str(b_params[k])) changed.append( { "experiment_id_prefix": key, "added_parameters": [{"name": k, "value": a_params[k]} for k in added_params], "removed_parameters": [{"name": k, "value": b_params[k]} for k in removed_params], - "changed_parameters": [ - {"name": k, "from": b_params[k], "to": a_params[k]} - for k in changed_params - ], + "changed_parameters": [{"name": k, "from": b_params[k], "to": a_params[k]} for k in changed_params], } ) return { @@ -1369,15 +1396,14 @@ def _proposal_diff_payload( "after": str(after_path), "before_hash": before_hash, "after_hash": after_hash, - "schema_version_match": ( - before.get("schema_version") == after.get("schema_version") - ), + "schema_version_match": (before.get("schema_version") == after.get("schema_version")), "added_candidates": added, "removed_candidates": removed, "changed_candidates": changed, "unchanged": False, } + def _proposal_diff_markdown(diff: dict[str, Any]) -> str: lines = [] lines.append("# proposal diff") @@ -1404,30 +1430,27 @@ def _proposal_diff_markdown(diff: dict[str, Any]) -> str: for entry in diff["changed_candidates"]: lines.append(f"### {entry['experiment_id_prefix']}") if entry["added_parameters"]: - added = ", ".join( - f"{p['name']}={p['value']}" for p in entry["added_parameters"] - ) + added = ", ".join(f"{p['name']}={p['value']}" for p in entry["added_parameters"]) lines.append(f"- added: {added}") if entry["removed_parameters"]: - removed = ", ".join( - f"{p['name']}={p['value']}" for p in entry["removed_parameters"] - ) + removed = ", ".join(f"{p['name']}={p['value']}" for p in entry["removed_parameters"]) lines.append(f"- removed: {removed}") if entry["changed_parameters"]: - changed = ", ".join( - f"{p['name']}: {p['from']}->{p['to']}" for p in entry["changed_parameters"] - ) + changed = ", ".join(f"{p['name']}: {p['from']}->{p['to']}" for p in entry["changed_parameters"]) lines.append(f"- changed: {changed}") lines.append("") return "\n".join(lines) + "\n" + def line_number(text: str, offset: int) -> int: return text.count("\n", 0, offset) + 1 + def default_derived_path(target: Path) -> Path: timestamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ") return target.with_name(f"{target.stem}.cursor-{timestamp}{target.suffix}") + def resolve_path_with_bases(raw_path: str, bases: list[Path]) -> Path: path = Path(raw_path) if path.is_absolute(): @@ -1438,6 +1461,7 @@ def resolve_path_with_bases(raw_path: str, bases: list[Path]) -> Path: return candidate return (bases[0] / path).resolve() + def validate_patch_safety(original: str, patched: str) -> list[dict[str, Any]]: errors = [] for code, pattern in FORBIDDEN_PATCH_PATTERNS: @@ -1453,6 +1477,7 @@ def validate_patch_safety(original: str, patched: str) -> list[dict[str, Any]]: ) return errors + def validate_patched_template_structure(original: str, patched: str) -> list[dict[str, Any]]: errors = [] structural_guards = ( @@ -1474,6 +1499,7 @@ def validate_patched_template_structure(original: str, patched: str) -> list[dic ) return errors + def evaluate_template_patch_request( request: dict[str, Any], *, @@ -1524,9 +1550,7 @@ def evaluate_template_patch_request( patched = original changes = request.get("changes", []) if not isinstance(changes, list) or not changes: - errors.append( - {"code": "no_changes", "message": "patch must include at least one change", "change_index": None} - ) + errors.append({"code": "no_changes", "message": "patch must include at least one change", "change_index": None}) changes = [] for index, change in enumerate(changes): match_context = str(change.get("match_context", "")) @@ -1630,4 +1654,109 @@ def evaluate_template_patch_request( ) return report -__all__ = [name for name in globals() if not name.startswith('__')] + +__all__ = [ + "EXPERIMENT_LEDGER_SCHEMA_VERSION", + "EXPERIMENT_STATUSES", + "FAILURE_PATTERNS", + "FORBIDDEN_PATCH_PATTERNS", + "REPO_ROOT", + "TEMPLATE_PATCH_SCHEMA_VERSION", + "Any", + "Path", + "_build_optimizer_space", + "_experiment_state_value", + "_objective_direction", + "_observations_from_states", + "_proposal_candidate_index", + "_proposal_diff_markdown", + "_proposal_diff_payload", + "_resolve_hyperband_config", + "annotations", + "append_experiment_update", + "append_jsonl", + "argparse", + "build_agent_session_report", + "build_circuit_breakers", + "build_objective_catalog", + "build_parameter_coverage", + "build_remaining_candidates", + "build_template_hints", + "build_trial_analysis", + "candidate_key", + "candidate_seen", + "choose_objective", + "classify_artifacts", + "classify_failure_text", + "config_contract_index", + "contextlib", + "copy_artifacts", + "decode_combination", + "default_derived_path", + "discover_finalize_logs", + "dt", + "effective_quality_target", + "effective_required_runs", + "error_code", + "evaluate_template_patch_request", + "experiment_id", + "extract_sbatch_directives", + "finalize_copy_candidates", + "finite_parameter_domains", + "hashlib", + "hb_engine", + "infer_experiment_shape", + "io", + "iter_candidate_parameters", + "itertools", + "json", + "latest_experiment_states", + "line_number", + "load_history_keys", + "load_json", + "load_ledger_keys", + "load_patch_request_from_descriptor", + "load_poll_statuses", + "load_validate_artifacts", + "materialize_record_config_patches", + "materialize_submit_wrapper", + "mean_or_none", + "normalize_candidate_parameters", + "normalize_config_patches", + "normalize_priority", + "objective_catalog", + "objective_index", + "parameter_index", + "parse_template_hint_line", + "patch_target_name", + "planned_submit_command", + "query_slurm_statuses", + "random", + "re", + "read_ledger_records", + "resolve_path_with_bases", + "run_id_from_log", + "run_validator", + "select_submit_candidates", + "shell_quote", + "short_hash", + "shutil", + "slurm_state_to_status", + "space_module", + "split_name_value", + "stable_hash", + "statistics", + "stdev_or_none", + "subprocess", + "summarize_json_file", + "summarize_optional_dirs", + "summarize_raw_dir", + "tpe_engine", + "utc_timestamp", + "validate_collected_artifacts", + "validate_finalize_run", + "validate_patch_safety", + "validate_patched_template_structure", + "value_allowed", + "write_json", +] diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/mlperf_rules.py b/plugins/profile-and-optimize/server/tools/ai_tuning/mlperf_rules.py index c570d6b..2ab420c 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/mlperf_rules.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/mlperf_rules.py @@ -37,12 +37,7 @@ from pathlib import Path from typing import Any -EXAMPLE_RULES_PATH = ( - Path(__file__).resolve().parents[2] - / "tuning" - / "examples" - / "mlperf-rules-v6-guardrails.json" -) +EXAMPLE_RULES_PATH = Path(__file__).resolve().parents[2] / "tuning" / "examples" / "mlperf-rules-v6-guardrails.json" @dataclass @@ -166,16 +161,11 @@ def validate_candidate( gbs_rules = bench_rules.get("global_batch_size") or {} gbs_min = _coerce_int(gbs_rules.get("min")) gbs_max = _coerce_int(gbs_rules.get("max")) - if (gbs_min is not None and gbs < gbs_min) or ( - gbs_max is not None and gbs > gbs_max - ): + if (gbs_min is not None and gbs < gbs_min) or (gbs_max is not None and gbs > gbs_max): out.append( RuleViolation( code="global_batch_outside_legal", - message=( - f"global_batch_size {gbs} outside legal range " - f"[{gbs_min},{gbs_max}]" - ), + message=(f"global_batch_size {gbs} outside legal range [{gbs_min},{gbs_max}]"), parameter="global_batch_size", observed=gbs, legal={"min": gbs_min, "max": gbs_max}, @@ -183,16 +173,12 @@ def validate_candidate( ) warmup = _coerce_int(parameters.get("warmup_steps")) - max_steps = _coerce_int(parameters.get("TRAINER_TRAIN_STEPS")) or _coerce_int( - parameters.get("max_steps") - ) + max_steps = _coerce_int(parameters.get("TRAINER_TRAIN_STEPS")) or _coerce_int(parameters.get("max_steps")) warmup_rules = bench_rules.get("warmup_steps") or {} if warmup is not None: wmin = _coerce_int(warmup_rules.get("min")) wmax = _coerce_int(warmup_rules.get("max")) - if (wmin is not None and warmup < wmin) or ( - wmax is not None and warmup > wmax - ): + if (wmin is not None and warmup < wmin) or (wmax is not None and warmup > wmax): out.append( RuleViolation( code="warmup_outside_legal", @@ -209,10 +195,7 @@ def validate_candidate( out.append( RuleViolation( code="warmup_above_proportion", - message=( - f"warmup_steps/max_steps={ratio:.4f} > limit " - f"{proportion_max}" - ), + message=(f"warmup_steps/max_steps={ratio:.4f} > limit {proportion_max}"), parameter="warmup_steps", observed=ratio, legal={"max_ratio": proportion_max}, @@ -250,16 +233,11 @@ def validate_candidate( if aux_coef is not None and aux_range: amin = _coerce_float(aux_range.get("min")) amax = _coerce_float(aux_range.get("max")) - if (amin is not None and aux_coef < amin) or ( - amax is not None and aux_coef > amax - ): + if (amin is not None and aux_coef < amin) or (amax is not None and aux_coef > amax): out.append( RuleViolation( code="aux_loss_coef_outside_legal", - message=( - f"AUX_LOSS_BALANCE_COEF={aux_coef} outside legal range " - f"[{amin},{amax}]" - ), + message=(f"AUX_LOSS_BALANCE_COEF={aux_coef} outside legal range [{amin},{amax}]"), parameter="AUX_LOSS_BALANCE_COEF", observed=aux_coef, legal={"min": amin, "max": amax}, @@ -272,10 +250,7 @@ def validate_candidate( out.append( RuleViolation( code="non_borrowable_hyperparameter_changed", - message=( - f"hyperparameter {parameter!r} is not borrowable for " - f"benchmark {benchmark!r}" - ), + message=(f"hyperparameter {parameter!r} is not borrowable for benchmark {benchmark!r}"), parameter=parameter, observed=parameters[parameter], ) @@ -284,9 +259,7 @@ def validate_candidate( return out -def validate_proposal( - rules: dict[str, Any], proposal: dict[str, Any], *, benchmark: str -) -> list[dict[str, Any]]: +def validate_proposal(rules: dict[str, Any], proposal: dict[str, Any], *, benchmark: str) -> list[dict[str, Any]]: candidates = proposal.get("candidates") or [] results: list[dict[str, Any]] = [] for index, candidate in enumerate(candidates): @@ -306,9 +279,7 @@ def validate_proposal( ) continue nexp = _coerce_int(params.get("NEXP")) or 1 - synthetic = ( - str(params.get("USE_SYNTHETIC_DATA", "0")).lower() in {"1", "true", "yes"} - ) + synthetic = str(params.get("USE_SYNTHETIC_DATA", "0")).lower() in {"1", "true", "yes"} violations = validate_candidate( rules, benchmark=benchmark, @@ -327,7 +298,8 @@ def validate_proposal( def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + description = (__doc__ or "MLPerf rule constraint validator").splitlines()[0] + parser = argparse.ArgumentParser(description=description) sub = parser.add_subparsers(dest="command", required=True) validate = sub.add_parser("validate", help="validate a proposal against the rules") diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/gp.py b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/gp.py index 15f6555..d3f5f5f 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/gp.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/gp.py @@ -29,7 +29,7 @@ def _matrix_zeros(n: int) -> list[list[float]]: def _rbf(a: list[float], b: list[float], length_scale: float, signal_variance: float) -> float: sq = 0.0 - for ai, bi in zip(a, b): + for ai, bi in zip(a, b, strict=True): diff = ai - bi sq += diff * diff return signal_variance * math.exp(-sq / max(2.0 * length_scale * length_scale, 1e-12)) @@ -113,7 +113,7 @@ def _log_marginal_likelihood( diag = max(L[index][index], 1e-12) log_det += math.log(diag) log_det *= 2.0 - data_fit = sum(t * a for t, a in zip(targets, alpha)) + data_fit = sum(t * a for t, a in zip(targets, alpha, strict=True)) return -0.5 * data_fit - 0.5 * log_det - 0.5 * n * math.log(2.0 * math.pi) @@ -180,10 +180,7 @@ def predict(model: GpModel, query: list[float]) -> tuple[float, float]: expected_dim = len(model.vectors[0]) if len(query) != expected_dim: raise ValueError(f"query dimension mismatch: expected {expected_dim}, got {len(query)}") - k_star = [ - _rbf(train, query, model.length_scale, model.signal_variance) - for train in model.vectors - ] + k_star = [_rbf(train, query, model.length_scale, model.signal_variance) for train in model.vectors] mean = sum(k_star[i] * model.alpha[i] for i in range(len(k_star))) v = _solve_lower(model.L, k_star) k_self = _rbf(query, query, model.length_scale, model.signal_variance) + JITTER @@ -292,4 +289,4 @@ def propose( return decoded, state -__all__ = ["GpModel", "fit", "predict", "expected_improvement", "propose"] +__all__ = ["GpModel", "expected_improvement", "fit", "predict", "propose"] diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/hyperband.py b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/hyperband.py index c1b1aa0..2d11ca9 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/hyperband.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/hyperband.py @@ -62,15 +62,15 @@ def plan(config: HyperbandConfig) -> list[Bracket]: eta = max(2, int(config.eta)) min_budget = max(1e-9, float(config.min_budget)) max_budget = max(min_budget * eta, float(config.max_budget)) - s_max = max(0, int(math.floor(_log(max_budget / min_budget, eta)))) + s_max = max(0, math.floor(_log(max_budget / min_budget, eta))) brackets: list[Bracket] = [] for s_index, s in enumerate(range(s_max, -1, -1)): - n_s = int(math.ceil((s_max + 1) / (s + 1) * (eta ** s))) + n_s = math.ceil((s_max + 1) / (s + 1) * (eta**s)) n_s = max(1, n_s) rungs: list[Rung] = [] for i in range(s + 1): - n_i = max(1, int(math.floor(n_s / (eta ** i)))) - budget_i = min_budget * (eta ** i) + n_i = max(1, math.floor(n_s / (eta**i))) + budget_i = min_budget * (eta**i) if budget_i > max_budget: budget_i = max_budget rungs.append( @@ -108,13 +108,18 @@ def to_dict(self) -> dict[str, object]: def from_dict(cls, data: dict[str, object] | None) -> HyperbandState: if not isinstance(data, dict): return cls() + raw_bracket_index = data.get("bracket_index", 0) + raw_rung_index = data.get("rung_index", 0) + raw_bracket_keys = data.get("bracket_keys", []) + bracket_keys = ( + [[str(key) for key in row] for row in raw_bracket_keys if isinstance(row, list)] + if isinstance(raw_bracket_keys, list) + else [] + ) return cls( - bracket_index=int(data.get("bracket_index", 0) or 0), - rung_index=int(data.get("rung_index", 0) or 0), - bracket_keys=[ - [str(key) for key in row] - for row in data.get("bracket_keys", []) or [] - ], + bracket_index=(int(raw_bracket_index) if isinstance(raw_bracket_index, (int, str)) else 0), + rung_index=(int(raw_rung_index) if isinstance(raw_rung_index, (int, str)) else 0), + bracket_keys=bracket_keys, completed=bool(data.get("completed", False)), ) @@ -210,10 +215,7 @@ def propose( sampled.append(params) sampled_keys = [_trial_key(params) for params in sampled] state.bracket_keys = [sampled_keys] - out_candidates = [ - {"parameters": params, "fidelity": rung.budget} - for params in sampled - ] + out_candidates = [{"parameters": params, "fidelity": rung.budget} for params in sampled] out_state.update(state.to_dict()) out_state.update( { @@ -235,7 +237,7 @@ def propose( key = _trial_key({k: decoded[k] for k in decoded}) if key in previous_keys: observed_pairs.append((key, float(o.value))) - needed = max(1, int(math.floor(len(previous_keys) / max(2, config.eta)))) + needed = max(1, math.floor(len(previous_keys) / max(2, config.eta))) survivors = _select_top(observed_pairs, needed, direction) if not survivors: # Without observed evidence, repeat the previous rung's keys at the new budget. @@ -252,10 +254,7 @@ def propose( name, _, value = chunk.partition("=") params[name] = value survivor_params.append(params) - out_candidates = [ - {"parameters": params, "fidelity": rung.budget} - for params in survivor_params - ] + out_candidates = [{"parameters": params, "fidelity": rung.budget} for params in survivor_params] out_state.update(state.to_dict()) out_state.update( { diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/space.py b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/space.py index 3499cdb..0cd3e5b 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/space.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/space.py @@ -173,9 +173,7 @@ def encode(self, parameters: dict[str, Any]) -> list[float]: def decode(self, vector: list[float]) -> dict[str, str]: if len(vector) != len(self.dimensions): - raise ValueError( - f"decode expected {len(self.dimensions)} values, got {len(vector)}" - ) + raise ValueError(f"decode expected {len(self.dimensions)} values, got {len(vector)}") return {dim.name: dim.decode(vector[index]) for index, dim in enumerate(self.dimensions)} def random(self, rng) -> dict[str, str]: @@ -192,7 +190,7 @@ def total_finite(self) -> int | None: return total def euclidean(self, a: list[float], b: list[float]) -> float: - return math.sqrt(sum((ai - bi) ** 2 for ai, bi in zip(a, b))) + return math.sqrt(sum((ai - bi) ** 2 for ai, bi in zip(a, b, strict=True))) __all__ = ["Dimension", "Space", "dimension_from_param"] diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/tpe.py b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/tpe.py index f2f05d5..89ed7c9 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/tpe.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/optimizer/tpe.py @@ -42,7 +42,7 @@ def _split_good_bad( else: sorted_obs = sorted(observations, key=lambda obs: obs.value) n = len(sorted_obs) - n_good = max(1, int(math.ceil(gamma * n))) + n_good = max(1, math.ceil(gamma * n)) n_good = min(n_good, n - 1) if n > 1 else 1 good = sorted_obs[:n_good] bad = sorted_obs[n_good:] or [sorted_obs[-1]] @@ -59,10 +59,7 @@ def _bandwidth(values: list[float]) -> float: if len(values) < 2: return 0.25 sorted_values = sorted(values) - diffs = [ - max(sorted_values[idx + 1] - sorted_values[idx], 0.0) - for idx in range(len(sorted_values) - 1) - ] + diffs = [max(sorted_values[idx + 1] - sorted_values[idx], 0.0) for idx in range(len(sorted_values) - 1)] width = max(diffs) if diffs else 0.0 span = sorted_values[-1] - sorted_values[0] if sorted_values else 0.0 return max(width, span / max(len(values), 1), 0.05) @@ -106,11 +103,7 @@ def propose( ) -> tuple[list[dict[str, str]], dict[str, object]]: """Return up to `limit` decoded candidate parameter dicts plus state metadata.""" - obs_list = [ - obs - for obs in observations - if isinstance(obs.value, (int, float)) and math.isfinite(obs.value) - ] + obs_list = [obs for obs in observations if isinstance(obs.value, (int, float)) and math.isfinite(obs.value)] state: dict[str, object] = { "strategy": "bayesian", "variant": "tpe", diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/proposals.py b/plugins/profile-and-optimize/server/tools/ai_tuning/proposals.py index 2f98fab..563fb0b 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/proposals.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/proposals.py @@ -2,10 +2,19 @@ from __future__ import annotations +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + try: # Package import, e.g. import tools.ai_tuning.proposals. - from .ai_tuning import ( + from .cmd_proposal import ( command_proposal_diff, command_proposal_validate, + ) + from .cmd_template_patch import ( command_template_patch_validate, ) from .helpers import ( @@ -23,9 +32,11 @@ validate_patched_template_structure, ) except ImportError: # Direct import from tools/ai_tuning. - from ai_tuning import ( + from cmd_proposal import ( command_proposal_diff, command_proposal_validate, + ) + from cmd_template_patch import ( command_template_patch_validate, ) from helpers import ( diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/test_ai_tuning.py b/plugins/profile-and-optimize/server/tools/ai_tuning/test_ai_tuning.py index 6cc6cea..5c1f58f 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/test_ai_tuning.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/test_ai_tuning.py @@ -25,6 +25,7 @@ B200_SPACE = REPO_ROOT / "tuning" / "examples" / "b200-offline.json" GB300_SPACE = REPO_ROOT / "tuning" / "examples" / "gb300-offline.json" + class UtcTimestampTest(unittest.TestCase): """Locks Python 3.10 compat for the UTC timestamp helpers. @@ -51,6 +52,7 @@ def test_default_derived_path_appends_compact_timestamp(self) -> None: self.assertTrue(derived.stem.startswith("foo.cursor-"), msg=str(derived)) self.assertTrue(derived.stem.endswith("Z"), msg=str(derived)) + class ProposalDiffTests(unittest.TestCase): """Tests for ai_tuning proposal diff PROPOSAL1 PROPOSAL2.""" @@ -190,6 +192,7 @@ def test_proposal_diff_markdown_includes_sections(self) -> None: if __name__ == "__main__": unittest.main() + class AiTuningCliTestPart1(unittest.TestCase): def test_operational_commands_require_explicit_space(self) -> None: cases = ( @@ -674,6 +677,7 @@ def test_proposal_validate_accepts_trainer_parameter_with_config_patch(self) -> self.assertTrue(result["contract_validation"]["objective_known"]) self.assertTrue(result["contract_validation"]["required_config_patches"][0]["provided"]) + class AiTuningCliTestPart2(unittest.TestCase): def make_finalize_fixture(self, root: Path, run_count: int = 5) -> tuple[Path, Path]: workdir = root / "workdir" @@ -951,7 +955,9 @@ def test_template_patch_rejects_removed_gres_guard(self) -> None: target = Path(tmp) / "run.sub" patch = Path(tmp) / "patch.json" output = Path(tmp) / "patch-report.json" - original = "#SBATCH --nodes=8\n#SBATCH --partition=\n#SBATCH --gres=gpu:b200:8\nenv -i bash run.sh\n" + original = ( + "#SBATCH --nodes=8\n#SBATCH --partition=\n#SBATCH --gres=gpu:b200:8\nenv -i bash run.sh\n" + ) target.write_text(original, encoding="utf-8") patch.write_text( json.dumps( @@ -1054,9 +1060,7 @@ def test_experiment_create_update_and_summary(self) -> None: self.assertEqual(rc, 0) summary = json.loads(summary_output.read_text(encoding="utf-8")) self.assertEqual(summary["experiment_count"], 2) - submitted = [ - item for item in summary["experiments"] if item["experiment_id"] == exp_id - ][0] + submitted = next(item for item in summary["experiments"] if item["experiment_id"] == exp_id) self.assertEqual(submitted["status"], "submitted") self.assertEqual(submitted["slurm_job_id"], "12345") @@ -1141,6 +1145,7 @@ def test_experiment_update_rejects_unknown_status(self) -> None: ] ) + class AiTuningCliTestPart3(unittest.TestCase): def make_finalize_fixture(self, root: Path, run_count: int = 5) -> tuple[Path, Path]: workdir = root / "workdir" @@ -1608,6 +1613,7 @@ def test_failure_classifier_detects_nccl_rdma_signature(self) -> None: self.assertIn("nccl_ib_devx_rtr_qp", codes) self.assertIn("nccl_ib_modify_qp_no_device", codes) + class AiTuningCliTestPart4(unittest.TestCase): def make_finalize_fixture(self, root: Path, run_count: int = 5) -> tuple[Path, Path]: workdir = root / "workdir" diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/test_mlperf_rules.py b/plugins/profile-and-optimize/server/tools/ai_tuning/test_mlperf_rules.py index ee9cdf1..01bf0b1 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/test_mlperf_rules.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/test_mlperf_rules.py @@ -10,7 +10,7 @@ if str(HERE) not in sys.path: sys.path.insert(0, str(HERE)) -from mlperf_rules import ( # noqa: E402 +from mlperf_rules import ( EXAMPLE_RULES_PATH, load_rules, parse_args, diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/test_modules.py b/plugins/profile-and-optimize/server/tools/ai_tuning/test_modules.py index dc39d49..c924117 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/test_modules.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/test_modules.py @@ -1,7 +1,10 @@ from __future__ import annotations import importlib.util +import os +import subprocess import sys +import tempfile import unittest from pathlib import Path @@ -25,6 +28,20 @@ def test_wrapper_and_split_modules_export_core_entrypoints(self) -> None: self.assertTrue(callable(proposals.command_proposal_validate)) self.assertTrue(safety.FORBIDDEN_PATCH_PATTERNS) + def test_proposals_direct_import_from_ai_tuning_path(self) -> None: + env = os.environ.copy() + env["PYTHONPATH"] = str(HERE) + with tempfile.TemporaryDirectory() as workdir: + proc = subprocess.run( + [sys.executable, "-S", "-c", "import proposals"], + cwd=workdir, + env=env, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + if __name__ == "__main__": unittest.main() diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/test_optimizer.py b/plugins/profile-and-optimize/server/tools/ai_tuning/test_optimizer.py index 221faa7..7ed30ff 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/test_optimizer.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/test_optimizer.py @@ -25,7 +25,7 @@ SPEC.loader.exec_module(ai_tuning) -from optimizer import gp, hyp_format, hyp_session, hyperband, space, tpe # noqa: E402 +from optimizer import gp, hyp_format, hyp_session, hyperband, space, tpe REPO_ROOT = Path(__file__).resolve().parents[2] @@ -33,6 +33,7 @@ B200_SPACE = REPO_ROOT / "tuning" / "examples" / "b200-offline.json" GB300_SPACE = REPO_ROOT / "tuning" / "examples" / "gb300-offline.json" + def _make_space(values_per_dim: int = 5) -> space.Space: dims = [ space.Dimension( @@ -50,12 +51,14 @@ def _make_space(values_per_dim: int = 5) -> space.Space: ] return space.Space(dimensions=dims) + def _quadratic(point: dict[str, str]) -> float: """Maximized at (0.7, 0.3).""" x = float(point["x"]) y = float(point["y"]) return -((x - 0.7) ** 2 + (y - 0.3) ** 2) + class SpaceTest(unittest.TestCase): def test_categorical_round_trip(self) -> None: dim = space.Dimension(name="k", kind="categorical", values=["a", "b", "c"]) @@ -81,9 +84,11 @@ def test_total_finite(self) -> None: ) self.assertEqual(finite_space.total_finite(), 2 * 3) + if __name__ == "__main__": unittest.main() + class TpeTest(unittest.TestCase): def test_cold_start_returns_random_until_min_observations(self) -> None: rng = random.Random(0) @@ -110,9 +115,7 @@ def test_tpe_outperforms_random_on_quadratic(self) -> None: tpe.Observation(vector=opt_space.encode(rand_params), value=_quadratic(rand_params)) ) tpe_params = opt_space.random(rng_tpe) - tpe_observations.append( - tpe.Observation(vector=opt_space.encode(tpe_params), value=_quadratic(tpe_params)) - ) + tpe_observations.append(tpe.Observation(vector=opt_space.encode(tpe_params), value=_quadratic(tpe_params))) for _ in range(20): tpe_candidates, _ = tpe.propose( @@ -124,9 +127,7 @@ def test_tpe_outperforms_random_on_quadratic(self) -> None: limit=1, ) params = tpe_candidates[0] - tpe_observations.append( - tpe.Observation(vector=opt_space.encode(params), value=_quadratic(params)) - ) + tpe_observations.append(tpe.Observation(vector=opt_space.encode(params), value=_quadratic(params))) rand_params = opt_space.random(rng_random) random_observations.append( tpe.Observation(vector=opt_space.encode(rand_params), value=_quadratic(rand_params)) @@ -136,15 +137,17 @@ def test_tpe_outperforms_random_on_quadratic(self) -> None: best_tpe = max(o.value for o in tpe_observations) self.assertGreaterEqual(best_tpe, best_random) + if __name__ == "__main__": unittest.main() + class GpTest(unittest.TestCase): def test_predict_recovers_observed_values(self) -> None: vectors = [[0.1, 0.2], [0.5, 0.4], [0.9, 0.8]] targets = [0.5, 1.5, -0.5] model = gp.fit(vectors, targets, length_scale_candidates=(0.3,), noise_variance=1e-6) - for vector, target in zip(vectors, targets): + for vector, target in zip(vectors, targets, strict=True): mean, _variance = gp.predict(model, vector) self.assertAlmostEqual(mean, target, places=2) @@ -165,9 +168,7 @@ def test_gp_outperforms_random_on_quadratic(self) -> None: tpe.Observation(vector=opt_space.encode(rand_params), value=_quadratic(rand_params)) ) gp_params = opt_space.random(rng_gp) - gp_observations.append( - tpe.Observation(vector=opt_space.encode(gp_params), value=_quadratic(gp_params)) - ) + gp_observations.append(tpe.Observation(vector=opt_space.encode(gp_params), value=_quadratic(gp_params))) for _ in range(15): gp_candidates, _ = gp.propose( @@ -179,9 +180,7 @@ def test_gp_outperforms_random_on_quadratic(self) -> None: limit=1, ) params = gp_candidates[0] - gp_observations.append( - tpe.Observation(vector=opt_space.encode(params), value=_quadratic(params)) - ) + gp_observations.append(tpe.Observation(vector=opt_space.encode(params), value=_quadratic(params))) rand_params = opt_space.random(rng_random) random_observations.append( tpe.Observation(vector=opt_space.encode(rand_params), value=_quadratic(rand_params)) @@ -191,9 +190,11 @@ def test_gp_outperforms_random_on_quadratic(self) -> None: best_gp = max(o.value for o in gp_observations) self.assertGreater(best_gp, best_random - 0.05) + if __name__ == "__main__": unittest.main() + class HyperbandTest(unittest.TestCase): def test_plan_produces_brackets(self) -> None: config = hyperband.HyperbandConfig(eta=3, min_budget=1, max_budget=27) @@ -253,9 +254,11 @@ def test_advance_rung_uses_observations(self) -> None: self.assertEqual(next_state["current_rung"], 1) self.assertGreater(next_state["current_budget"], 1.0) + if __name__ == "__main__": unittest.main() + class HypFormatTest(unittest.TestCase): def test_parses_real_hyp_fixture(self) -> None: fixture = FIXTURES / "hyp" / "scaled_run.tp-comm-overlap-true.hyp" @@ -268,9 +271,11 @@ def test_parses_real_hyp_fixture(self) -> None: kinds = {entry["name"]: entry["kind"] for entry in manifest} self.assertEqual(kinds["TENSOR_MODEL_PARALLEL"], "integer") + if __name__ == "__main__": unittest.main() + class HypSessionTest(unittest.TestCase): def test_imports_synthetic_session(self) -> None: session_dir = FIXTURES / "hyp-session" @@ -290,9 +295,11 @@ def test_imports_synthetic_session(self) -> None: self.assertAlmostEqual(scores[0], 10.875, places=3) self.assertAlmostEqual(scores[-1], 12.100, places=3) + if __name__ == "__main__": unittest.main() + class CliTest(unittest.TestCase): def test_optimizer_status_smoke(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -484,14 +491,14 @@ def test_optimizer_propose_state_out_round_trip_advances_rung(self) -> None: "--seed", "3", ] - rc = ai_tuning.main(base_args + ["--state-out", str(state_file), "--output", str(first_output)]) + rc = ai_tuning.main([*base_args, "--state-out", str(state_file), "--output", str(first_output)]) self.assertEqual(rc, 0) self.assertTrue(state_file.is_file()) first = json.loads(first_output.read_text(encoding="utf-8")) self.assertEqual(first["optimizer_state"]["current_rung"], 0) self.assertTrue(first["optimizer_state"]["bracket_keys"]) - rc = ai_tuning.main(base_args + ["--state-file", str(state_file), "--output", str(second_output)]) + rc = ai_tuning.main([*base_args, "--state-file", str(state_file), "--output", str(second_output)]) self.assertEqual(rc, 0) second = json.loads(second_output.read_text(encoding="utf-8")) self.assertEqual(second["optimizer_state"]["current_rung"], 1) @@ -863,5 +870,6 @@ def test_optimizer_end_to_end_bayesian_pipeline(self) -> None: self.assertEqual(compared["direction"], "maximize") self.assertEqual(compared["winner"], "compare-b") + if __name__ == "__main__": unittest.main() diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/test_safety.py b/plugins/profile-and-optimize/server/tools/ai_tuning/test_safety.py index 01e778a..67bba68 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/test_safety.py +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/test_safety.py @@ -56,11 +56,7 @@ def test_benign_snippets_do_not_false_fire(self) -> None: # uses \b after sbatch. for snippet in benign: with self.subTest(snippet=snippet): - hits = [ - code - for code, pattern in safety.FORBIDDEN_PATCH_PATTERNS - if re.search(pattern, snippet) - ] + hits = [code for code, pattern in safety.FORBIDDEN_PATCH_PATTERNS if re.search(pattern, snippet)] # benign cases should not match the *submit* pattern. self.assertNotIn("submit_slurm_job", hits, snippet) diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/copied-config.sh b/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/copied-config.sh index 7e3333e..f03af81 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/copied-config.sh +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/copied-config.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + export NCCL_DEBUG=WARN export NEXP=1 export DATADIR=/datasets/example diff --git a/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/raw-run-dir/config_example.sh b/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/raw-run-dir/config_example.sh index 9478531..91f45b2 100644 --- a/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/raw-run-dir/config_example.sh +++ b/plugins/profile-and-optimize/server/tools/ai_tuning/testdata/raw-run-dir/config_example.sh @@ -1 +1,3 @@ +#!/usr/bin/env bash + export EXAMPLE_CONFIG=1 diff --git a/plugins/profile-and-optimize/server/tools/capacity_sizing.py b/plugins/profile-and-optimize/server/tools/capacity_sizing.py index b7a89b1..a8ce2e9 100644 --- a/plugins/profile-and-optimize/server/tools/capacity_sizing.py +++ b/plugins/profile-and-optimize/server/tools/capacity_sizing.py @@ -24,6 +24,7 @@ Worked cross-check (the "3M TPM, 18 pods" thread): per-replica 2916 tok/s at c=128 (tok/s/user 22.78), 8-GPU pod, util=1.0 -> ceil(50000/2916) = 18 pods = 144 GPUs. """ + from __future__ import annotations import argparse @@ -31,6 +32,7 @@ import json import math import sys +from itertools import pairwise from pathlib import Path GPU_HOURLY_USD = 8.6 @@ -50,12 +52,12 @@ def per_replica_tps(self) -> float: @dataclasses.dataclass class SizingRow: sla_toks_per_user: float - concurrency: float # c* (the max concurrency meeting the SLA) + concurrency: float # c* (the max concurrency meeting the SLA) per_replica_tps: float replicas: int gpus: int usd_per_month: float - regime: str # measured-anchor | interp | below-knee-plateau | latency-tier-infeasible + regime: str # measured-anchor | interp | below-knee-plateau | latency-tier-infeasible @dataclasses.dataclass @@ -89,17 +91,18 @@ def parse_anchors(spec: str) -> list[Anchor]: raise ValueError("need at least one (concurrency:toks_per_user) anchor") anchors.sort(key=lambda a: a.concurrency) # tok/s/user must be non-increasing in concurrency (more concurrent streams -> <= per-user rate) - for lo, hi in zip(anchors, anchors[1:]): + for lo, hi in pairwise(anchors): if hi.toks_per_user > lo.toks_per_user + 1e-9: raise ValueError( f"non-monotonic curve: tok/s/user rose from {lo.toks_per_user} @c{lo.concurrency} " - f"to {hi.toks_per_user} @c{hi.concurrency} (must be non-increasing in concurrency)") + f"to {hi.toks_per_user} @c{hi.concurrency} (must be non-increasing in concurrency)" + ) return anchors def _interp_concurrency_for_sla(anchors: list[Anchor], sla: float) -> tuple[float, float, str]: """Return (c*, per_replica_tps, regime) for a target SLA via log-c piecewise interpolation.""" - hi_tpu = anchors[0].toks_per_user # largest tok/s/user (lowest concurrency) + hi_tpu = anchors[0].toks_per_user # largest tok/s/user (lowest concurrency) lo_tpu = anchors[-1].toks_per_user # smallest tok/s/user (highest concurrency = knee) if sla > hi_tpu + 1e-9: @@ -112,7 +115,7 @@ def _interp_concurrency_for_sla(anchors: list[Anchor], sla: float) -> tuple[floa return a.concurrency, a.per_replica_tps, "below-knee-plateau" # bracket: find adjacent anchors with tpu[i] >= sla >= tpu[i+1] - for lo, hi in zip(anchors, anchors[1:]): + for lo, hi in pairwise(anchors): if lo.toks_per_user + 1e-9 >= sla >= hi.toks_per_user - 1e-9: if abs(lo.toks_per_user - sla) < 1e-9: return lo.concurrency, lo.per_replica_tps, "measured-anchor" @@ -120,7 +123,7 @@ def _interp_concurrency_for_sla(anchors: list[Anchor], sla: float) -> tuple[floa return hi.concurrency, hi.per_replica_tps, "measured-anchor" frac = (lo.toks_per_user - sla) / (lo.toks_per_user - hi.toks_per_user) log_c = math.log10(lo.concurrency) + frac * (math.log10(hi.concurrency) - math.log10(lo.concurrency)) - c = 10.0 ** log_c + c = 10.0**log_c per_replica = c * sla return c, per_replica, "interp" # unreachable given the guards above @@ -161,27 +164,37 @@ def resolve( notes: list[str] = [] if any(r.regime == "interp" for r in rows): - notes.append("interp rows are log-c interpolated between measured anchors (DRAFT); feed the full " - "roofline sweep (c=8..128) for a VERDICT.") + notes.append( + "interp rows are log-c interpolated between measured anchors (DRAFT); feed the full " + "roofline sweep (c=8..128) for a VERDICT." + ) if any(r.regime == "below-knee-plateau" for r in rows): - notes.append("below-knee SLAs share the knee per-replica throughput (throughput plateaus past the " - "knee; the workload is host/KV-bound there), so the pod count is flat.") + notes.append( + "below-knee SLAs share the knee per-replica throughput (throughput plateaus past the " + "knee; the workload is host/KV-bound there), so the pod count is flat." + ) if any(r.regime == "latency-tier-infeasible" for r in rows): - notes.append("an SLA above the c=1 anchor is a latency-tier point; serving a large TPM there needs " - "an impractical pod count.") + notes.append( + "an SLA above the c=1 anchor is a latency-tier point; serving a large TPM there needs " + "an impractical pod count." + ) return SizingResult(model, tpm, tok_per_s, gpus_per_pod, util, gpu_hourly_usd, anchors, rows, notes) def render_md(r: SizingResult) -> str: lines = [f"# Capacity sizing (SLA-first): {r.model}", ""] - lines.append(f"TPM target {r.tpm:,.0f} output tok/min ({r.tok_per_s_target:,.0f} tok/s); " - f"pod = {r.gpus_per_pod} GPUs; util {r.util:.0%}; ${r.gpu_hourly_usd}/GPU-hour.") + lines.append( + f"TPM target {r.tpm:,.0f} output tok/min ({r.tok_per_s_target:,.0f} tok/s); " + f"pod = {r.gpus_per_pod} GPUs; util {r.util:.0%}; ${r.gpu_hourly_usd}/GPU-hour." + ) lines.append("") lines.append("| SLA tok/s/user | c* | per-pod tok/s | pods | GPUs | $/month | regime |") lines.append("| ---: | ---: | ---: | ---: | ---: | ---: | --- |") for row in r.rows: - lines.append(f"| {row.sla_toks_per_user:g} | {row.concurrency:g} | {row.per_replica_tps:,.0f} | " - f"{row.replicas} | {row.gpus} | ${row.usd_per_month:,.0f} | {row.regime} |") + lines.append( + f"| {row.sla_toks_per_user:g} | {row.concurrency:g} | {row.per_replica_tps:,.0f} | " + f"{row.replicas} | {row.gpus} | ${row.usd_per_month:,.0f} | {row.regime} |" + ) if r.notes: lines += ["", "## Notes"] lines += [f"- {n}" for n in r.notes] @@ -192,10 +205,12 @@ def render_md(r: SizingResult) -> str: def build_arg_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description="SLA-first capacity sizing (TPM + tok/s/user SLA -> pods/GPUs).") p.add_argument("--tpm", type=float, required=True, help="total output tokens per minute target (e.g. 3000000)") - p.add_argument("--sla", required=True, - help="tokens/s/user SLA(s), comma-separated (e.g. '20,50,100' or a single '50')") - p.add_argument("--anchors", required=True, - help="measured curve as 'c:tok_per_user,...' (e.g. '1:215.5,10:122.2,256:41.8')") + p.add_argument( + "--sla", required=True, help="tokens/s/user SLA(s), comma-separated (e.g. '20,50,100' or a single '50')" + ) + p.add_argument( + "--anchors", required=True, help="measured curve as 'c:tok_per_user,...' (e.g. '1:215.5,10:122.2,256:41.8')" + ) p.add_argument("--gpus-per-pod", type=int, default=4, help="GPUs per replica/pod (GB300 TP4 node = 4)") p.add_argument("--util", type=float, default=0.70, help="target utilization headroom (default 0.70)") p.add_argument("--gpu-hourly-usd", type=float, default=GPU_HOURLY_USD) @@ -208,8 +223,15 @@ def main(argv: list[str] | None = None) -> int: args = build_arg_parser().parse_args(argv) anchors = parse_anchors(args.anchors) sla_list = [float(s) for s in str(args.sla).split(",") if s.strip()] - r = resolve(tpm=args.tpm, sla_list=sla_list, anchors=anchors, gpus_per_pod=args.gpus_per_pod, - util=args.util, gpu_hourly_usd=args.gpu_hourly_usd, model=args.model) + r = resolve( + tpm=args.tpm, + sla_list=sla_list, + anchors=anchors, + gpus_per_pod=args.gpus_per_pod, + util=args.util, + gpu_hourly_usd=args.gpu_hourly_usd, + model=args.model, + ) md = render_md(r) sys.stdout.write(md) if args.emit: diff --git a/plugins/profile-and-optimize/server/tools/evidence/test_evidence.py b/plugins/profile-and-optimize/server/tools/evidence/test_evidence.py index ec46cdb..f070984 100644 --- a/plugins/profile-and-optimize/server/tools/evidence/test_evidence.py +++ b/plugins/profile-and-optimize/server/tools/evidence/test_evidence.py @@ -11,7 +11,6 @@ from tools.evidence.evidence_cli import CONTRACT, _operator_path, build_parser, main - SERVER_ROOT = Path(__file__).resolve().parents[2] BASH = shutil.which("bash") or "/bin/bash" diff --git a/plugins/profile-and-optimize/server/tools/findings/findings_cli.py b/plugins/profile-and-optimize/server/tools/findings/findings_cli.py index 47d7902..3a6b305 100644 --- a/plugins/profile-and-optimize/server/tools/findings/findings_cli.py +++ b/plugins/profile-and-optimize/server/tools/findings/findings_cli.py @@ -14,29 +14,47 @@ import argparse import json import sys -from datetime import datetime, timezone +from collections.abc import Iterable +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Iterable +from typing import Any, Protocol + + +class _YamlModule(Protocol): + def safe_load(self, stream: str) -> Any: ... + + def safe_dump( + self, + data: Any, + *, + sort_keys: bool, + default_flow_style: bool, + ) -> str: ... + # yaml is required for round-tripping; install via the bundled server's `dev` extras. try: - import yaml + import yaml as _yaml except ImportError: # pragma: no cover - yaml = None # type: ignore[assignment] + _yaml = None VALID_SEVERITIES = ("critical", "high", "medium", "low", "informational") VALID_STATUSES = ("open", "in_progress", "resolved") -def _require_yaml() -> None: - if yaml is None: - print("FATAL: PyYAML not available. Install via `bash plugins/profile-and-optimize/server/install.sh --with-dev`.", file=sys.stderr) +def _require_yaml() -> _YamlModule: + if _yaml is None: + print( + "FATAL: PyYAML not available. Install via `bash plugins/profile-and-optimize/server/install.sh --with-dev`.", + file=sys.stderr, + ) sys.exit(2) + return _yaml def _load_findings(path: Path) -> dict[str, Any]: - _require_yaml() + yaml = _require_yaml() if not path.exists(): return {"findings": []} raw = path.read_text() @@ -50,7 +68,7 @@ def _load_findings(path: Path) -> dict[str, Any]: def _dump_findings(path: Path, data: dict[str, Any]) -> None: - _require_yaml() + yaml = _require_yaml() path.write_text(yaml.safe_dump(data, sort_keys=False, default_flow_style=False)) @@ -92,7 +110,7 @@ def _record(args: argparse.Namespace) -> int: finding["affected_entities"].append({"kind": kind, "value": value}) if args.notes: finding["notes"] = args.notes - finding["detected_at_utc"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + finding["detected_at_utc"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") errors = _validate_finding(finding) if errors: @@ -155,7 +173,9 @@ def _flush(severity: str) -> None: counters[bucket] += 1 num = f"{prefix}{counters[bucket]}" src = f"{f.get('source_skill', '?')} / {f.get('source_query', '?')}" - lines.append(f"| {num} | {f.get('headline', '?')} | {src} | {f.get('recommended_action', '?')} | {f.get('status', 'open')} |") + lines.append( + f"| {num} | {f.get('headline', '?')} | {src} | {f.get('recommended_action', '?')} | {f.get('status', 'open')} |" + ) lines.append("") for sev in ("critical", "high", "medium", "low", "informational"): @@ -238,11 +258,19 @@ def _diff(args: argparse.Namespace) -> int: def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="findings", description="Structured findings library: record + render + diff.") + parser = argparse.ArgumentParser( + prog="findings", description="Structured findings library: record + render + diff." + ) sub = parser.add_subparsers(dest="command", required=True) - rec = sub.add_parser("record", help="Append a finding to a bundle's findings.yaml.", description="Append a finding to a bundle's findings.yaml.") - rec.add_argument("--findings-yaml", required=True, help="Path to the findings.yaml file (will be created if missing)") + rec = sub.add_parser( + "record", + help="Append a finding to a bundle's findings.yaml.", + description="Append a finding to a bundle's findings.yaml.", + ) + rec.add_argument( + "--findings-yaml", required=True, help="Path to the findings.yaml file (will be created if missing)" + ) rec.add_argument("--id", required=True) rec.add_argument("--severity", required=True, choices=VALID_SEVERITIES) rec.add_argument("--source-skill", required=True) @@ -251,18 +279,32 @@ def build_parser() -> argparse.ArgumentParser: rec.add_argument("--recommended-action", required=True) rec.add_argument("--status", choices=VALID_STATUSES, default="open") rec.add_argument("--evidence-path", default=None) - rec.add_argument("--affected-entity", action="append", default=None, metavar="KIND=VALUE", help="May be passed multiple times (e.g. --affected-entity zone=)") + rec.add_argument( + "--affected-entity", + action="append", + default=None, + metavar="KIND=VALUE", + help="May be passed multiple times (e.g. --affected-entity zone=)", + ) rec.add_argument("--notes", default=None) rec.add_argument("--json", action="store_true") rec.set_defaults(func=_record) - rdr = sub.add_parser("render", help="Convert findings.yaml -> findings.md.", description="Convert findings.yaml -> findings.md (table grouped by severity).") + rdr = sub.add_parser( + "render", + help="Convert findings.yaml -> findings.md.", + description="Convert findings.yaml -> findings.md (table grouped by severity).", + ) rdr.add_argument("--findings-yaml", required=True) rdr.add_argument("--out", default=None, help="Write markdown to this path; default stdout") rdr.add_argument("--json", action="store_true") rdr.set_defaults(func=_render) - dff = sub.add_parser("diff", help="Compare two findings.yaml files + emit drift report.", description="Compare two findings.yaml files and emit a markdown drift report.") + dff = sub.add_parser( + "diff", + help="Compare two findings.yaml files + emit drift report.", + description="Compare two findings.yaml files and emit a markdown drift report.", + ) dff.add_argument("--baseline", required=True) dff.add_argument("--current", required=True) dff.add_argument("--out", default=None, help="Write markdown to this path; default stdout") @@ -281,7 +323,15 @@ def main(argv: Iterable[str] | None = None) -> int: CONTRACT: dict[str, dict] = { "record": { "safety": "writes_artifacts", - "required": ("--findings-yaml", "--id", "--severity", "--source-skill", "--source-query", "--headline", "--recommended-action"), + "required": ( + "--findings-yaml", + "--id", + "--severity", + "--source-skill", + "--source-query", + "--headline", + "--recommended-action", + ), "optional": ("--status", "--evidence-path", "--affected-entity", "--notes", "--json"), "json": True, "ack": None, diff --git a/plugins/profile-and-optimize/server/tools/findings/test_findings.py b/plugins/profile-and-optimize/server/tools/findings/test_findings.py index 83c5464..c29f594 100644 --- a/plugins/profile-and-optimize/server/tools/findings/test_findings.py +++ b/plugins/profile-and-optimize/server/tools/findings/test_findings.py @@ -3,7 +3,6 @@ from __future__ import annotations import argparse -import json from pathlib import Path import pytest @@ -86,7 +85,9 @@ def test_diff_detects_new_resolved_and_status_changes(tmp_path: Path) -> None: f._record(_record_args(tmp_path, findings_yaml=str(base), id="a", severity="critical", headline="A")) f._record(_record_args(tmp_path, findings_yaml=str(base), id="b", severity="high", headline="B")) # current: drop b, add c, mark a as in_progress - f._record(_record_args(tmp_path, findings_yaml=str(cur), id="a", severity="critical", headline="A", status="in_progress")) + f._record( + _record_args(tmp_path, findings_yaml=str(cur), id="a", severity="critical", headline="A", status="in_progress") + ) f._record(_record_args(tmp_path, findings_yaml=str(cur), id="c", severity="medium", headline="C")) args = argparse.Namespace( diff --git a/plugins/profile-and-optimize/server/tools/known_good_config/known_good_config_cli.py b/plugins/profile-and-optimize/server/tools/known_good_config/known_good_config_cli.py index 442e0bc..6bb6dea 100644 --- a/plugins/profile-and-optimize/server/tools/known_good_config/known_good_config_cli.py +++ b/plugins/profile-and-optimize/server/tools/known_good_config/known_good_config_cli.py @@ -41,9 +41,19 @@ "safety": "writes_artifacts", "required": ("--model",), "optional": ( - "--registry", "--slug", "--arch", "--hardware", "--engine", - "--required-flag", "--champion-config-ref", "--champion-verdict", - "--champion-campaign", "--fallback", "--grind-frontier", "--notes", "--json", + "--registry", + "--slug", + "--arch", + "--hardware", + "--engine", + "--required-flag", + "--champion-config-ref", + "--champion-verdict", + "--champion-campaign", + "--fallback", + "--grind-frontier", + "--notes", + "--json", ), "json": True, "ack": None, @@ -53,7 +63,10 @@ "safety": "read_only", "required": ("--model",), "optional": ( - "--registry", "--serve-args", "--deploy-file", "--json", + "--registry", + "--serve-args", + "--deploy-file", + "--json", ), "json": True, "ack": None, @@ -129,7 +142,7 @@ def _parse_required_flag(raw: str) -> dict[str, str]: parts = [p.strip() for p in raw.split("|")] keys = ("flag", "match", "severity", "why", "affected", "evidence") out: dict[str, str] = {} - for key, val in zip(keys, parts): + for key, val in zip(keys, parts, strict=False): if val: out[key] = val if "flag" not in out: @@ -144,9 +157,7 @@ def _validate_severity(severity: object, *, source: str) -> str: """Return a documented severity or stop before unsafe warn-only behavior.""" if not isinstance(severity, str) or severity not in ALLOWED_SEVERITIES: allowed = ", ".join(sorted(ALLOWED_SEVERITIES)) - raise SystemExit( - f"FATAL: {source} severity must be one of {allowed}; got {severity!r}" - ) + raise SystemExit(f"FATAL: {source} severity must be one of {allowed}; got {severity!r}") return severity @@ -257,13 +268,15 @@ def cmd_check(args: argparse.Namespace) -> int: try: found = re.search(match, text) is not None except re.error as exc: - raise SystemExit(f"FATAL: bad `match` regex {match!r} for model {args.model}: {exc}") + raise SystemExit(f"FATAL: bad `match` regex {match!r} for model {args.model}: {exc}") from exc if not found: - missing.append({ - "flag": rf.get("flag", match), - "severity": severity, - "why": rf.get("why", ""), - }) + missing.append( + { + "flag": rf.get("flag", match), + "severity": severity, + "why": rf.get("why", ""), + } + ) hard = [m for m in missing if m["severity"] in FAIL_SEVERITIES] verdict = "fail" if hard else "pass" @@ -291,16 +304,22 @@ def build_parser() -> argparse.ArgumentParser: record = sub.add_parser("record", description=CONTRACT["record"]["description"]) record.add_argument("--model", required=True, help="HF id / served-model-name (the registry key)") - record.add_argument("--registry", default=None, help="Path to known-good-configs.yaml (else $KNOWN_GOOD_CONFIG_REGISTRY / walk-up)") + record.add_argument( + "--registry", default=None, help="Path to known-good-configs.yaml (else $KNOWN_GOOD_CONFIG_REGISTRY / walk-up)" + ) record.add_argument("--slug", default=None, help="Deploy-bundle slug (-deploy)") record.add_argument("--arch", default=None, help="Short arch note (why the quirk exists)") record.add_argument("--hardware", default=None, help="Validated hardware (e.g. GB300 TP=4 NVFP4)") record.add_argument("--engine", default=None, help="vllm | sglang") record.add_argument( - "--required-flag", action="append", default=None, + "--required-flag", + action="append", + default=None, help="Pipe-delimited flag|match|severity|why|affected|evidence (repeatable)", ) - record.add_argument("--champion-config-ref", default=None, help="Path to the full champion config (my-values / deploy yaml)") + record.add_argument( + "--champion-config-ref", default=None, help="Path to the full champion config (my-values / deploy yaml)" + ) record.add_argument("--champion-verdict", default=None, help="DRAFT | VERDICT ") record.add_argument("--champion-campaign", default=None, help="perf-lake campaign id (evidence)") record.add_argument("--fallback", default=None, help="A known-working alternative") @@ -311,9 +330,13 @@ def build_parser() -> argparse.ArgumentParser: check = sub.add_parser("check", description=CONTRACT["check"]["description"]) check.add_argument("--model", required=True, help="HF id / served-model-name to check") - check.add_argument("--registry", default=None, help="Path to known-good-configs.yaml (else $KNOWN_GOOD_CONFIG_REGISTRY / walk-up)") + check.add_argument( + "--registry", default=None, help="Path to known-good-configs.yaml (else $KNOWN_GOOD_CONFIG_REGISTRY / walk-up)" + ) check_input = check.add_mutually_exclusive_group(required=True) - check_input.add_argument("--serve-args", default=None, help="The joined serve-args string to scan for required flags") + check_input.add_argument( + "--serve-args", default=None, help="The joined serve-args string to scan for required flags" + ) check_input.add_argument("--deploy-file", default=None, help="A deploy/values file to scan for required flags") check.add_argument( "--require-registered", diff --git a/plugins/profile-and-optimize/server/tools/known_good_config/test_known_good_config.py b/plugins/profile-and-optimize/server/tools/known_good_config/test_known_good_config.py index 7e9fbd7..e8a0999 100644 --- a/plugins/profile-and-optimize/server/tools/known_good_config/test_known_good_config.py +++ b/plugins/profile-and-optimize/server/tools/known_good_config/test_known_good_config.py @@ -6,7 +6,6 @@ from __future__ import annotations -import textwrap from pathlib import Path import pytest @@ -72,9 +71,7 @@ def test_operator_path_rejects_nul_bytes() -> None: def test_build_parser_accepts_json_for_both_verbs() -> None: parser = build_parser() assert parser.parse_args(["record", "--model", "m", "--json"]).json is True - assert parser.parse_args( - ["check", "--model", "m", "--serve-args", "", "--json"] - ).json is True + assert parser.parse_args(["check", "--model", "m", "--serve-args", "", "--json"]).json is True def test_parse_required_flag_defaults() -> None: @@ -100,23 +97,35 @@ def test_parse_required_flag_rejects_unknown_severity(severity: str) -> None: def test_check_pass_when_required_flag_present(registry: Path, capsys) -> None: - rc = main([ - "check", "--registry", str(registry), - "--model", "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4", - "--serve-args", "vllm serve --additional-config '{\"gdn_prefill_backend\":\"triton\"}'", - "--json", - ]) + rc = main( + [ + "check", + "--registry", + str(registry), + "--model", + "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4", + "--serve-args", + 'vllm serve --additional-config \'{"gdn_prefill_backend":"triton"}\'', + "--json", + ] + ) assert rc == 0 assert '"verdict": "pass"' in capsys.readouterr().out def test_check_fail_when_boot_blocker_missing(registry: Path, capsys) -> None: - rc = main([ - "check", "--registry", str(registry), - "--model", "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4", - "--serve-args", "vllm serve --tensor-parallel-size 4", # no gdn_prefill_backend - "--json", - ]) + rc = main( + [ + "check", + "--registry", + str(registry), + "--model", + "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4", + "--serve-args", + "vllm serve --tensor-parallel-size 4", # no gdn_prefill_backend + "--json", + ] + ) assert rc == 1 # fail-closed out = capsys.readouterr().out assert '"verdict": "fail"' in out @@ -124,12 +133,18 @@ def test_check_fail_when_boot_blocker_missing(registry: Path, capsys) -> None: def test_check_perf_severity_missing_is_warning_not_failure(registry: Path, capsys) -> None: - rc = main([ - "check", "--registry", str(registry), - "--model", "zai-org/GLM-5.1", - "--serve-args", "vllm serve --tensor-parallel-size 8", # no FLASHMLA (perf only) - "--json", - ]) + rc = main( + [ + "check", + "--registry", + str(registry), + "--model", + "zai-org/GLM-5.1", + "--serve-args", + "vllm serve --tensor-parallel-size 8", # no FLASHMLA (perf only) + "--json", + ] + ) assert rc == 0 # perf-severity missing -> warn, not fail out = capsys.readouterr().out assert '"verdict": "pass"' in out @@ -137,19 +152,36 @@ def test_check_perf_severity_missing_is_warning_not_failure(registry: Path, caps def test_check_require_registered_fails_for_unknown_model(registry: Path, capsys) -> None: - rc = main([ - "check", "--registry", str(registry), - "--model", "made/up-model", "--serve-args", "", "--require-registered", "--json", - ]) + rc = main( + [ + "check", + "--registry", + str(registry), + "--model", + "made/up-model", + "--serve-args", + "", + "--require-registered", + "--json", + ] + ) assert rc == 1 assert "model_not_registered" in capsys.readouterr().out def test_check_unknown_model_fails_without_compatibility_flag(registry: Path, capsys) -> None: - rc = main([ - "check", "--registry", str(registry), "--model", "made/up-model", - "--serve-args", "", "--json", - ]) + rc = main( + [ + "check", + "--registry", + str(registry), + "--model", + "made/up-model", + "--serve-args", + "", + "--json", + ] + ) assert rc == 1 output = capsys.readouterr().out assert '"registered": false' in output @@ -163,19 +195,25 @@ def test_check_requires_exactly_one_input(registry: Path) -> None: with pytest.raises(SystemExit): parser.parse_args(common) with pytest.raises(SystemExit): - parser.parse_args(common + ["--serve-args", "vllm serve", "--deploy-file", "deploy.yaml"]) + parser.parse_args([*common, "--serve-args", "vllm serve", "--deploy-file", "deploy.yaml"]) def test_check_rejects_unknown_registry_severity(registry: Path) -> None: registry.write_text(REGISTRY_HEADER.replace("severity: perf", "severity: informational")) with pytest.raises(SystemExit, match="severity must be one of"): - main([ - "check", "--registry", str(registry), - "--model", "zai-org/GLM-5.1", - "--serve-args", "vllm serve", - "--json", - ]) + main( + [ + "check", + "--registry", + str(registry), + "--model", + "zai-org/GLM-5.1", + "--serve-args", + "vllm serve", + "--json", + ] + ) # --------------------------------------------------------------------------- @@ -184,13 +222,22 @@ def test_check_rejects_unknown_registry_severity(registry: Path) -> None: def test_record_appends_new_model_and_preserves_comments(registry: Path) -> None: - rc = main([ - "record", "--registry", str(registry), - "--model", "Qwen/Qwen3-235B-A22B-Thinking-2507", - "--slug", "qwen3-235b-thinking", "--engine", "vllm", - "--champion-verdict", "VERDICT vLLM 1.6x SGLang", - "--json", - ]) + rc = main( + [ + "record", + "--registry", + str(registry), + "--model", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + "--slug", + "qwen3-235b-thinking", + "--engine", + "vllm", + "--champion-verdict", + "VERDICT vLLM 1.6x SGLang", + "--json", + ] + ) assert rc == 0 text = registry.read_text() assert "# HAND-CURATED -- a comment that record() MUST preserve." in text @@ -205,20 +252,32 @@ def test_record_appends_new_model_and_preserves_comments(registry: Path) -> None def test_record_rejects_existing_model(registry: Path) -> None: - rc = main([ - "record", "--registry", str(registry), - "--model", "zai-org/GLM-5.1", "--json", - ]) + rc = main( + [ + "record", + "--registry", + str(registry), + "--model", + "zai-org/GLM-5.1", + "--json", + ] + ) assert rc == 2 # must edit by hand def test_record_parses_required_flag_tuple(registry: Path) -> None: - rc = main([ - "record", "--registry", str(registry), - "--model", "new/model", - "--required-flag", "--moe-backend=cutlass|moe-backend[ =]+cutlass|crash-high-c|default MoE crashes c>=64|vllm 0.22|some/evidence", - "--json", - ]) + rc = main( + [ + "record", + "--registry", + str(registry), + "--model", + "new/model", + "--required-flag", + "--moe-backend=cutlass|moe-backend[ =]+cutlass|crash-high-c|default MoE crashes c>=64|vllm 0.22|some/evidence", + "--json", + ] + ) assert rc == 0 reg = _load_registry(registry) entry = _find_model(reg, "new/model") diff --git a/plugins/profile-and-optimize/server/tools/loader_advisor.py b/plugins/profile-and-optimize/server/tools/loader_advisor.py index 789398f..cac8a98 100644 --- a/plugins/profile-and-optimize/server/tools/loader_advisor.py +++ b/plugins/profile-and-optimize/server/tools/loader_advisor.py @@ -23,6 +23,7 @@ s3fs FUSE was RETIRED 2026-06-09 (~50min single-stream, ~4x slower than hf-pull): the S3-only + no-runai-image case now returns `none` instead of the slow s3fs fallback. """ + from __future__ import annotations import argparse @@ -62,18 +63,26 @@ def resolve( """Encode the loader decision tree. Inputs are booleans knowable from the tier's serve config (mtp) + cluster/image facts (the other three).""" gates = [ - Gate("mtp", "info", - "tier uses an MTP / in-checkpoint spec-decode drafter" if mtp - else "no in-checkpoint spec-decode drafter"), - Gate("hf_egress", "pass" if hf_egress_ok else "fail", - "HuggingFace egress available at (re)start" if hf_egress_ok - else "no HuggingFace egress (air-gap / policy)"), - Gate("runai_image", "pass" if image_has_runai else "fail", - "serve image carries runai_model_streamer" if image_has_runai - else "serve image lacks runai_model_streamer"), - Gate("s3", "pass" if s3_available else "fail", - "S3 S3 mirror + creds available" if s3_available - else "no S3 S3 mirror"), + Gate( + "mtp", + "info", + "tier uses an MTP / in-checkpoint spec-decode drafter" if mtp else "no in-checkpoint spec-decode drafter", + ), + Gate( + "hf_egress", + "pass" if hf_egress_ok else "fail", + "HuggingFace egress available at (re)start" if hf_egress_ok else "no HuggingFace egress (air-gap / policy)", + ), + Gate( + "runai_image", + "pass" if image_has_runai else "fail", + "serve image carries runai_model_streamer" if image_has_runai else "serve image lacks runai_model_streamer", + ), + Gate( + "s3", + "pass" if s3_available else "fail", + "S3 S3 mirror + creds available" if s3_available else "no S3 S3 mirror", + ), ] reasons: list[str] = [] rec, tier, needs_patch = "none", "none", False @@ -84,12 +93,14 @@ def resolve( reasons.append( "non-MTP tier: RunAI streamer streams the S3 byte-identical weights to GPU " "(~10min, no HF dependency, no 700Gi emptyDir, no privileged FUSE) -- the clean win " - "over hf-pull.") + "over hf-pull." + ) elif hf_egress_ok: rec, tier = "hf-pull", "ok" reasons.append( "non-MTP but RunAI is unavailable (no runai-capable image and/or no S3 mirror) " - "-> hf-pull (~12-13min via xet; needs HF egress + a 700Gi emptyDir).") + "-> hf-pull (~12-13min via xet; needs HF egress + a 700Gi emptyDir)." + ) # (s3fs FUSE fallback retired 2026-06-09: S3-only + no runai image now -> none) else: # MTP tier if hf_egress_ok: @@ -97,19 +108,22 @@ def resolve( reasons.append( "MTP tier: hf-pull is MTP-native (~12-13min). RunAI needs the drafter patch AND " "double-streams the checkpoint (~16min), so hf-pull is preferred when HF egress " - "is available.") + "is available." + ) elif image_has_runai and s3_available: rec, tier, needs_patch = "runai", "tradeoff", True reasons.append( "MTP + no HF egress -> RunAI streamer + the MTP-drafter patch (S3, no HF " "dependency) but ~16min due to the drafter double-stream. Tradeoff: " - "provenance/air-gap over cold-start speed.") + "provenance/air-gap over cold-start speed." + ) # (s3fs FUSE fallback retired 2026-06-09: S3-only + no runai image now -> none) if rec == "none": reasons.append( "No viable loader for these constraints: need at least HF egress, or a S3 mirror " - "plus a S3-capable loader. Provide --hf-egress yes, or --s3 yes (+ a runai image).") + "plus a S3-capable loader. Provide --hf-egress yes, or --s3 yes (+ a runai image)." + ) return LoaderResult(rec, LOADER_FRAGMENT.get(rec, ""), needs_patch, tier, gates, reasons) @@ -127,8 +141,8 @@ def render_md(r: LoaderResult, mtp: bool) -> str: else: patch = " + MTP-drafter patch" if r.needs_mtp_patch else "" lines.append( - f"**RECOMMENDED LOADER: `{r.recommended}`{patch} [{r.tier}]** " - f"(deploy fragment: `{r.fragment_key}`)") + f"**RECOMMENDED LOADER: `{r.recommended}`{patch} [{r.tier}]** (deploy fragment: `{r.fragment_key}`)" + ) lines += ["", "## Inputs / gates"] for g in r.gates: mark = {"pass": "[pass]", "fail": "[fail]", "info": "[info]"}[g.status] @@ -137,9 +151,12 @@ def render_md(r: LoaderResult, mtp: bool) -> str: for x in r.reasons: lines.append(f"- {x}") if r.needs_mtp_patch: - lines += ["", "## Note", - "- The runai+MTP path requires the speculative.py drafter patch " - "(the runai-mtp-drafter speculative.py patch)."] + lines += [ + "", + "## Note", + "- The runai+MTP path requires the speculative.py drafter patch " + "(the runai-mtp-drafter speculative.py patch).", + ] return "\n".join(lines) + "\n" @@ -148,14 +165,14 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--mtp", dest="mtp", action="store_true", help="tier uses an MTP/spec-decode drafter") p.add_argument("--no-mtp", dest="mtp", action="store_false") p.set_defaults(mtp=None) - p.add_argument("--serve-args", default="", - help="vLLM serve args; auto-detects MTP when --mtp/--no-mtp not given") + p.add_argument("--serve-args", default="", help="vLLM serve args; auto-detects MTP when --mtp/--no-mtp not given") p.add_argument("--hf-egress", choices=["yes", "no"], default="yes") p.add_argument("--image-has-runai", choices=["yes", "no"], default="yes") p.add_argument("--s3", choices=["yes", "no"], default="yes") p.add_argument("--emit", default="", help="directory to write loader_advisor.json + LOADER.md") - p.add_argument("--print-fragment", action="store_true", - help="print only the fragment_key (for the scaffolder) and exit") + p.add_argument( + "--print-fragment", action="store_true", help="print only the fragment_key (for the scaffolder) and exit" + ) args = p.parse_args(argv) mtp = args.mtp if args.mtp is not None else mtp_from_serve_args(args.serve_args) diff --git a/plugins/profile-and-optimize/server/tools/perf_baseline/helpers.py b/plugins/profile-and-optimize/server/tools/perf_baseline/helpers.py index a20f901..9c6d032 100644 --- a/plugins/profile-and-optimize/server/tools/perf_baseline/helpers.py +++ b/plugins/profile-and-optimize/server/tools/perf_baseline/helpers.py @@ -16,7 +16,6 @@ from pathlib import Path from typing import Any - _SAFE_SEGMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/aggregator.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/aggregator.py index 7fb5844..a07d684 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/aggregator.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/aggregator.py @@ -21,9 +21,9 @@ from __future__ import annotations import json +from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path -from typing import Iterable from tools.perf_tune_report.coverage import CoverageSummary, summarize from tools.perf_tune_report.schema import AtlasCell, write_jsonl @@ -74,9 +74,7 @@ def aggregate(campaign_dir: Path) -> AggregateResult: try: rows.append(AtlasCell(**item)) except (TypeError, ValueError) as exc: - raise ValueError( - f"FATAL: row {idx} in {path} fails AtlasCell schema: {exc}" - ) from exc + raise ValueError(f"FATAL: row {idx} in {path} fails AtlasCell schema: {exc}") from exc atlas_path = campaign_dir / "atlas.jsonl" write_jsonl(rows, atlas_path) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/capture_signature.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/capture_signature.py index a0a6bd4..233d06d 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/capture_signature.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/capture_signature.py @@ -11,10 +11,11 @@ import hashlib import json import shutil +from collections.abc import Iterable from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Iterable +from typing import Any from tools.perf_tune_report.schema import AtlasCell, read_jsonl @@ -167,11 +168,7 @@ def _normalize_value(v: Any) -> Any: def _pick_extra(extra: dict[str, Any]) -> dict[str, Any]: - return { - k: _normalize_value(extra[k]) - for k in _EXTRA_KEYS - if k in extra and extra[k] not in (None, "", "unknown") - } + return {k: _normalize_value(extra[k]) for k in _EXTRA_KEYS if k in extra and extra[k] not in (None, "", "unknown")} def _variant_fields(row: AtlasCell) -> dict[str, Any]: @@ -208,9 +205,7 @@ def _variant_fields(row: AtlasCell) -> dict[str, Any]: else extra.get("prefix_cache") ), "bench_backend": getattr(row, "bench_backend", "") or "", - "max_num_batched_tokens": ( - int(row.max_num_batched_tokens) if row.max_num_batched_tokens is not None else None - ), + "max_num_batched_tokens": (int(row.max_num_batched_tokens) if row.max_num_batched_tokens is not None else None), "cudagraph_mode": row.cudagraph_mode, "gpu_memory_utilization": row.gpu_memory_utilization, "cache_mode": row.cache_mode, @@ -274,10 +269,14 @@ def _artifact_cell_id(campaign_dir: Path, row: AtlasCell) -> str: """ arm = (row.extra or {}).get("arm") - if isinstance(arm, str) and arm and ( - (campaign_dir / "cells" / arm).is_dir() - or (campaign_dir / "cells" / f"{arm}-decode").is_dir() - or (campaign_dir / "cells" / f"{arm}-prefill").is_dir() + if ( + isinstance(arm, str) + and arm + and ( + (campaign_dir / "cells" / arm).is_dir() + or (campaign_dir / "cells" / f"{arm}-decode").is_dir() + or (campaign_dir / "cells" / f"{arm}-prefill").is_dir() + ) ): return arm return row.cell_id @@ -299,8 +298,7 @@ def _artifact_target_path(campaign_dir: Path, cell_id: str, artifact: str) -> Pa def _cell_artifacts(campaign_dir: Path, cell_id: str) -> dict[str, bool]: - return {artifact: _artifact_path(campaign_dir, cell_id, artifact).is_file() - for artifact in CAPTURE_ARTIFACTS} + return {artifact: _artifact_path(campaign_dir, cell_id, artifact).is_file() for artifact in CAPTURE_ARTIFACTS} def collect_cells(campaign_dirs: Iterable[Path]) -> list[CellCaptureState]: @@ -379,7 +377,7 @@ def build_plan(target_campaign_dirs: list[Path], source_campaign_dirs: list[Path return CapturePlan( schema_version=PLAN_SCHEMA_VERSION, - generated_at_utc=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + generated_at_utc=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), target_campaigns=[str(p.resolve()) for p in target_campaign_dirs], source_campaigns=[str(p.resolve()) for p in source_campaign_dirs], cells=targets, @@ -395,8 +393,7 @@ def _load_plan(path: Path) -> dict[str, Any]: raise ValueError(f"capture reuse plan is not valid JSON: {path}: {exc}") from exc if data.get("schema_version") != PLAN_SCHEMA_VERSION: raise ValueError( - f"capture reuse plan schema mismatch: expected {PLAN_SCHEMA_VERSION}, " - f"got {data.get('schema_version')!r}" + f"capture reuse plan schema mismatch: expected {PLAN_SCHEMA_VERSION}, got {data.get('schema_version')!r}" ) return data @@ -408,7 +405,6 @@ def materialize_reuse(plan_path: Path, *, dry_run: bool = False) -> Materialized copied: list[dict[str, Any]] = [] skipped: list[dict[str, Any]] = [] for item in plan.get("reuse_candidates", []): - artifact = str(item["artifact"]) src = Path(item["source_path"]).expanduser() dst = Path(item["target_path"]).expanduser() if not src.is_file(): @@ -438,7 +434,7 @@ def materialize_reuse(plan_path: Path, *, dry_run: bool = False) -> Materialized if existing.get("schema_version") != REUSE_SCHEMA_VERSION: existing = { "schema_version": REUSE_SCHEMA_VERSION, - "generated_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "generated_at_utc": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "artifacts": [], } existing.setdefault("artifacts", []).append(record) @@ -446,7 +442,7 @@ def materialize_reuse(plan_path: Path, *, dry_run: bool = False) -> Materialized copied.append(record) return MaterializedReuse( schema_version=REUSE_SCHEMA_VERSION, - generated_at_utc=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + generated_at_utc=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), copied=copied, skipped=skipped, ) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/champion_select.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/champion_select.py index 12cf165..8528660 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/champion_select.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/champion_select.py @@ -34,6 +34,8 @@ from pathlib import Path from typing import Any +import yaml + from tools.perf_tune_report.schema import AtlasCell, read_jsonl SCHEMA_VERSION = "champion_select_v1" @@ -168,10 +170,14 @@ def _artifact_cell_id(campaign_dir: Path, row: AtlasCell | None, cell_id: str) - """ arm = (row.extra or {}).get("arm") if row is not None else None - if isinstance(arm, str) and arm and ( - (campaign_dir / "cells" / arm).is_dir() - or (campaign_dir / "cells" / f"{arm}-decode").is_dir() - or (campaign_dir / "cells" / f"{arm}-prefill").is_dir() + if ( + isinstance(arm, str) + and arm + and ( + (campaign_dir / "cells" / arm).is_dir() + or (campaign_dir / "cells" / f"{arm}-decode").is_dir() + or (campaign_dir / "cells" / f"{arm}-prefill").is_dir() + ) ): return arm return cell_id @@ -245,11 +251,7 @@ def _summarize_sol(campaign_dir: Path, cell_id: str, focus_c: int) -> SolSummary s.l4_kernel_count = len(nk["kernels"]) s.sol_rigor = ( - "L4" if s.l4_present - else "L3" if s.l3_present - else "L2" if s.l2_present - else "L1" if s.l1_present - else "none" + "L4" if s.l4_present else "L3" if s.l3_present else "L2" if s.l2_present else "L1" if s.l1_present else "none" ) return s @@ -260,17 +262,15 @@ def _load_roofline(campaign_dir: Path, cell_id: str) -> dict[str, Any] | None: ) -def _resolve_baseline(arm_ids: list[str], rows_by_cell: dict[str, dict[int, AtlasCell]], - explicit: str | None) -> str | None: +def _resolve_baseline( + arm_ids: list[str], rows_by_cell: dict[str, dict[int, AtlasCell]], explicit: str | None +) -> str | None: if explicit and explicit in rows_by_cell: return explicit # heuristic: a *-base / *-baseline cell, preferring vLLM (the incumbent). bases = [c for c in arm_ids if c.endswith(("-base", "-baseline"))] if bases: - vllm_bases = [ - c for c in bases - if any(r.serving_engine != "sglang" for r in rows_by_cell[c].values()) - ] + vllm_bases = [c for c in bases if any(r.serving_engine != "sglang" for r in rows_by_cell[c].values())] return sorted(vllm_bases or bases)[0] return sorted(arm_ids)[0] if arm_ids else None @@ -283,9 +283,8 @@ def _campaign_focus(campaign_dir: Path) -> str | None: yml = campaign_dir / "config.yaml" if yml.is_file(): try: - import yaml # type: ignore data = yaml.safe_load(yml.read_text()) or {} - except Exception: + except (AttributeError, OSError, TypeError, ValueError, yaml.YAMLError): data = {} focus = (data.get("campaign") or {}).get("focus") or data.get("focus") if focus: @@ -331,9 +330,7 @@ def select( ) -> ChampionResult: atlas_path = campaign_dir / "atlas.jsonl" if not atlas_path.is_file(): - raise FileNotFoundError( - f"atlas.jsonl not found at {atlas_path}; run `perftunereport atlas_aggregate` first." - ) + raise FileNotFoundError(f"atlas.jsonl not found at {atlas_path}; run `perftunereport atlas_aggregate` first.") rows = read_jsonl(atlas_path) if not rows: raise ValueError(f"atlas.jsonl is empty at {atlas_path}") @@ -357,9 +354,7 @@ def select( rows_by_cell.setdefault(r.cell_id, {})[r.concurrency] = r arm_ids = sorted(rows_by_cell) if not arm_ids: - raise ValueError( - f"no rankable arm cells in {atlas_path} (only roofline cells found)" - ) + raise ValueError(f"no rankable arm cells in {atlas_path} (only roofline cells found)") hardware = rows[0].hardware tensor_parallel = rows[0].tensor_parallel @@ -380,10 +375,7 @@ def _metric_value(cell: AtlasCell | None) -> float | None: base_row = _row_at_c(baseline_cell) if baseline_cell else None base_tpot = base_row.tpot_median_ms if base_row else None base_metric = _metric_value(base_row) - slo_ms = ( - slo_abs_ms if slo_abs_ms is not None - else (base_tpot * slo_rel if base_tpot is not None else None) - ) + slo_ms = slo_abs_ms if slo_abs_ms is not None else (base_tpot * slo_rel if base_tpot is not None else None) def _slo_verdict(cell: AtlasCell | None, is_base: bool) -> str: if is_base: @@ -415,12 +407,8 @@ def _mk_variant(cell_id: str) -> VariantRow: tpot_median_ms=cell.tpot_median_ms if cell else None, pct_win_vs_baseline=0.0 if is_base else _pct_win(val), slo_verdict=_slo_verdict(cell, is_base), - sol=_summarize_sol( - campaign_dir, _artifact_cell_id(campaign_dir, cell, cell_id), resolved_focus_c - ), - has_roofline=_load_roofline( - campaign_dir, _artifact_cell_id(campaign_dir, cell, cell_id) - ) is not None, + sol=_summarize_sol(campaign_dir, _artifact_cell_id(campaign_dir, cell, cell_id), resolved_focus_c), + has_roofline=_load_roofline(campaign_dir, _artifact_cell_id(campaign_dir, cell, cell_id)) is not None, ) # Rank the non-baseline arms: SLO-passing first, then by the focus metric @@ -446,14 +434,11 @@ def _sort_key(v: VariantRow): # Recommendation: the best SLO-passing variant that actually beats the # baseline on the focus metric; else keep the baseline (no-change). champion = next( - (v for v in top_rows if v.slo_verdict == "PASS-SLO" - and (v.pct_win_vs_baseline or 0) > 0), + (v for v in top_rows if v.slo_verdict == "PASS-SLO" and (v.pct_win_vs_baseline or 0) > 0), None, ) recommended_cell = champion.cell_id if champion else baseline_cell - recommended_engine = champion.engine if champion else ( - variants[0].engine if variants else None - ) + recommended_engine = champion.engine if champion else (variants[0].engine if variants else None) # Roofline overlay: baseline + every selected variant that carries a sweep. overlay: dict[str, Any] = {} @@ -516,26 +501,29 @@ def _evaluate_gates( # Variance: same-node + >=3 trials (the DRAFT->VERDICT rule). if trials is None and not same_node: - gates.append(GateResult("variance", "unknown", - "trials / same-node not supplied (pass --trials >=3 --same-node)")) + gates.append( + GateResult("variance", "unknown", "trials / same-node not supplied (pass --trials >=3 --same-node)") + ) elif same_node and (trials or 0) >= 3: gates.append(GateResult("variance", "pass", f"same-node, {trials} trials")) else: - gates.append(GateResult("variance", "fail", - f"need same-node + >=3 trials (got same_node={same_node}, trials={trials})")) + gates.append( + GateResult("variance", "fail", f"need same-node + >=3 trials (got same_node={same_node}, trials={trials})") + ) # Multi-workload coverage. if not workloads_present: - gates.append(GateResult("multi_workload", "unknown", - "no --workloads-present supplied; cannot confirm the canonical suite")) + gates.append( + GateResult( + "multi_workload", "unknown", "no --workloads-present supplied; cannot confirm the canonical suite" + ) + ) else: missing = [w for w in require_workloads if w not in set(workloads_present)] if missing: - gates.append(GateResult("multi_workload", "fail", - f"missing workloads: {','.join(missing)}")) + gates.append(GateResult("multi_workload", "fail", f"missing workloads: {','.join(missing)}")) else: - gates.append(GateResult("multi_workload", "pass", - f"covered: {','.join(require_workloads)}")) + gates.append(GateResult("multi_workload", "pass", f"covered: {','.join(require_workloads)}")) # Accuracy. # Name the measured serving-quality metrics (import_model_eval eval_acc) in the detail so @@ -554,19 +542,23 @@ def _evaluate_gates( elif accuracy_gate == "fail": gates.append(GateResult("accuracy", "fail", "accuracy gate FAILED -> do-not-ship" + eval_detail)) else: - gates.append(GateResult( - "accuracy", "unknown", - "accuracy gate not run (--accuracy-gate / --accuracy-floor)" + eval_detail)) + gates.append( + GateResult( + "accuracy", "unknown", "accuracy gate not run (--accuracy-gate / --accuracy-floor)" + eval_detail + ) + ) # DCGM byte-grounding (L3) of the recommended champion. if recommended is None: gates.append(GateResult("dcgm_grounded", "unknown", "no champion selected")) elif recommended.sol.l3_present: - gates.append(GateResult("dcgm_grounded", "pass", - f"champion sol_rigor={recommended.sol.sol_rigor}")) + gates.append(GateResult("dcgm_grounded", "pass", f"champion sol_rigor={recommended.sol.sol_rigor}")) else: - gates.append(GateResult("dcgm_grounded", "fail", - "champion has no L3 DCGM byte-grounding (run dcgm_correlate / roofline-sweep)")) + gates.append( + GateResult( + "dcgm_grounded", "fail", "champion has no L3 DCGM byte-grounding (run dcgm_correlate / roofline-sweep)" + ) + ) return gates @@ -584,10 +576,7 @@ def render_markdown(result: ChampionResult, *, title: str | None = None) -> str: lines: list[str] = [] lines.append(f"# {title or 'Champion selection'} -- {result.campaign_id}") lines.append("") - lines.append( - f"**RECOMMENDED FOR PRODUCTION: `{rec}` ({result.recommended_engine or '?'}) " - f"[{tier}]**" - ) + lines.append(f"**RECOMMENDED FOR PRODUCTION: `{rec}` ({result.recommended_engine or '?'}) [{tier}]**") lines.append("") lines.append( f"- focus: `{result.focus}` | metric: `{metric_label}` | " @@ -608,8 +597,11 @@ def render_markdown(result: ChampionResult, *, title: str | None = None) -> str: lines.append("|" + "---|" * 11) for v in result.variants: tag = " (baseline)" if v.is_baseline else "" - win = "--" if v.is_baseline else (f"{v.pct_win_vs_baseline:+.1f}%" - if v.pct_win_vs_baseline is not None else "n/a") + win = ( + "--" + if v.is_baseline + else (f"{v.pct_win_vs_baseline:+.1f}%" if v.pct_win_vs_baseline is not None else "n/a") + ) lines.append( f"| `{v.cell_id}`{tag} | {v.engine} | {_fmt(v.focus_metric)} | {win} | " f"{_fmt(v.tpot_median_ms)} | {v.slo_verdict} | {v.sol.sol_rigor} | " @@ -639,8 +631,9 @@ def render_markdown(result: ChampionResult, *, title: str | None = None) -> str: return "\n".join(lines) -def write_outputs(result: ChampionResult, campaign_dir: Path, *, - out_md: Path | None = None, title: str | None = None) -> tuple[Path, Path]: +def write_outputs( + result: ChampionResult, campaign_dir: Path, *, out_md: Path | None = None, title: str | None = None +) -> tuple[Path, Path]: json_path = campaign_dir / "champion_select.json" md_path = out_md or (campaign_dir / "CHAMPION.md") json_path.write_text(json.dumps(result.to_dict(), indent=2, sort_keys=True) + "\n") diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/coverage.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/coverage.py index b8ada63..4cc26ed 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/coverage.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/coverage.py @@ -16,8 +16,8 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass -from typing import Iterable from tools.perf_tune_report.schema import ( STATUS_EVICTED, @@ -83,11 +83,7 @@ def summarize(rows: Iterable[AtlasCell]) -> CoverageSummary: partial = sum(1 for s in cell_status.values() if s == STATUS_PARTIAL) failed = sum(1 for s in cell_status.values() if s == STATUS_FAILED) evicted = sum(1 for s in cell_status.values() if s == STATUS_EVICTED) - non_plot_ready_full = sum( - 1 - for cid, s in cell_status.items() - if s == STATUS_FULL and not cell_has_plottable[cid] - ) + non_plot_ready_full = sum(1 for cid, s in cell_status.items() if s == STATUS_FULL and not cell_has_plottable[cid]) return CoverageSummary( atlas_cells=full + partial + failed, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/dcgm_correlate.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/dcgm_correlate.py index d65123e..f308c28 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/dcgm_correlate.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/dcgm_correlate.py @@ -41,13 +41,11 @@ from __future__ import annotations import json -import math from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any, Protocol - _DCGM_CORRELATION_OUTPUT = "dcgm_correlation.json" @@ -79,6 +77,7 @@ class PrometheusClient(Protocol): def query_instant(self, promql: str, ts: datetime) -> list[TimeSeries]: """One-shot point query (used for label/cardinality probes).""" + ... def query_range( self, @@ -88,6 +87,7 @@ def query_range( step_s: float, ) -> list[TimeSeries]: """Range query returning one TimeSeries per result label combo.""" + ... @dataclass @@ -112,11 +112,11 @@ def duration_s(self) -> float: class DcgmQuery: """One pre-built PromQL query for one peak's DCGM proxy.""" - peak_key: str # e.g. "hbm3e_tbps" - metric: str # e.g. "DCGM_FI_PROF_DRAM_ACTIVE" - promql: str # the actual query string - unit: str # "ratio" | "bytes_per_scrape" | "bytes" - is_fallback: bool # True if this is the DCGM_FI_DEV_* fallback + peak_key: str # e.g. "hbm3e_tbps" + metric: str # e.g. "DCGM_FI_PROF_DRAM_ACTIVE" + promql: str # the actual query string + unit: str # "ratio" | "bytes_per_scrape" | "bytes" + is_fallback: bool # True if this is the DCGM_FI_DEV_* fallback metrics_combined: list[str] = field(default_factory=list) # for TX+RX-style @@ -133,8 +133,8 @@ class ResourceResult: measured_tflops_avg: float | None peak_per_gpu: float peak_per_gpu_units: str - peak_aggregate: float # peak_per_gpu * n_gpus (in same units as peak) - sol_pct: float | None # 0..100; None if measurement unavailable + peak_aggregate: float # peak_per_gpu * n_gpus (in same units as peak) + sol_pct: float | None # 0..100; None if measurement unavailable notes: list[str] = field(default_factory=list) @@ -178,7 +178,7 @@ class DcgmCorrelationResult: sweep_end_utc: str duration_s: float n_gpus: int - dcgm_group_level: str # "prof" | "counter" | "absent" + dcgm_group_level: str # "prof" | "counter" | "absent" scrape_interval_s: float | None short_sweep_warning: bool resources: list[ResourceResult] @@ -265,34 +265,40 @@ def build_queries( if use_prof and "dcgm_metrics_bytes" in entry: # Paired byte counters (TX + RX). Sum them. metrics = entry["dcgm_metrics_bytes"] - sum_terms = " + ".join(f'rate({m}{label_filter}[1m])' for m in metrics) - queries.append(DcgmQuery( - peak_key=peak_key, - metric=f"sum({','.join(metrics)})", - promql=f"sum by (gpu) ({sum_terms})", - unit="bytes_per_s", # rate(counter_bytes[1m]) yields bytes/s - is_fallback=False, - metrics_combined=metrics, - )) + sum_terms = " + ".join(f"rate({m}{label_filter}[1m])" for m in metrics) + queries.append( + DcgmQuery( + peak_key=peak_key, + metric=f"sum({','.join(metrics)})", + promql=f"sum by (gpu) ({sum_terms})", + unit="bytes_per_s", # rate(counter_bytes[1m]) yields bytes/s + is_fallback=False, + metrics_combined=metrics, + ) + ) elif use_prof and "dcgm_metric" in entry: metric = entry["dcgm_metric"] unit = entry.get("dcgm_unit", "ratio") - queries.append(DcgmQuery( - peak_key=peak_key, - metric=metric, - promql=f"avg by (gpu) ({metric}{label_filter})", - unit=unit, - is_fallback=False, - )) + queries.append( + DcgmQuery( + peak_key=peak_key, + metric=metric, + promql=f"avg by (gpu) ({metric}{label_filter})", + unit=unit, + is_fallback=False, + ) + ) elif not use_prof and "dcgm_fallback_metric" in entry: metric = entry["dcgm_fallback_metric"] - queries.append(DcgmQuery( - peak_key=peak_key, - metric=metric, - promql=f"avg by (gpu) ({metric}{label_filter})", - unit="counter", - is_fallback=True, - )) + queries.append( + DcgmQuery( + peak_key=peak_key, + metric=metric, + promql=f"avg by (gpu) ({metric}{label_filter})", + unit="counter", + is_fallback=True, + ) + ) return queries @@ -567,9 +573,7 @@ def cross_attribute_zymtrace( duration_s = result.duration_s # Aggregate FLOPS across the sweep across all GPUs. compute_total_flops = ( - (compute_avg_tflops_per_gpu * 1e12 * duration_s * n_gpus) - if compute_avg_tflops_per_gpu is not None - else None + (compute_avg_tflops_per_gpu * 1e12 * duration_s * n_gpus) if compute_avg_tflops_per_gpu is not None else None ) out: list[PerCategoryAttribution] = [] @@ -584,7 +588,6 @@ def cross_attribute_zymtrace( ceiling_metric_key = cat_info.get("metric") ceiling_entry = hw_data.get(ceiling_metric_key) if ceiling_metric_key else None ceiling_value = ceiling_entry.get("value") if isinstance(ceiling_entry, dict) else None - ceiling_units = ceiling_entry.get("units") if isinstance(ceiling_entry, dict) else None # Pick the workload byte/FLOP source per category bound. if bound == "bandwidth": @@ -628,18 +631,20 @@ def cross_attribute_zymtrace( peak_tflops = ceiling_value * 1000.0 sol_pct_compute = (effective_tflops / peak_tflops) * 100.0 if peak_tflops > 0 else None - out.append(PerCategoryAttribution( - category=category, - time_share_pct=time_share_pct, - attributed_bytes_total=attributed_bytes, - attributed_flops_total=attributed_flops, - effective_bw_during_category_window=effective_bw, - effective_tflops_during_category_window=effective_tflops, - sol_pct_bw=sol_pct_bw, - sol_pct_compute=sol_pct_compute, - bound=bound, - ceiling_metric=ceiling_metric_key, - )) + out.append( + PerCategoryAttribution( + category=category, + time_share_pct=time_share_pct, + attributed_bytes_total=attributed_bytes, + attributed_flops_total=attributed_flops, + effective_bw_during_category_window=effective_bw, + effective_tflops_during_category_window=effective_tflops, + sol_pct_bw=sol_pct_bw, + sol_pct_compute=sol_pct_compute, + bound=bound, + ceiling_metric=ceiling_metric_key, + ) + ) return out @@ -680,9 +685,7 @@ def _query_mean_power_watts_per_gpu( label_filter = _build_label_filter(inputs) promql = f"avg by (gpu) ({_POWER_METRIC}{label_filter})" try: - series = client.query_range( - promql, inputs.sweep_start, inputs.sweep_end, step_s=step_s - ) + series = client.query_range(promql, inputs.sweep_start, inputs.sweep_end, step_s=step_s) except Exception: # noqa: BLE001 - power is best-effort; never break correlate return None, promql vals = [v for s in (series or []) for (_ts, v) in s.samples if v is not None and v > 0] @@ -735,13 +738,15 @@ def correlate( node_set: set[str] = set() for q in queries: - query_records.append({ - "peak_key": q.peak_key, - "metric": q.metric, - "promql": q.promql, - "unit": q.unit, - "is_fallback": q.is_fallback, - }) + query_records.append( + { + "peak_key": q.peak_key, + "metric": q.metric, + "promql": q.promql, + "unit": q.unit, + "is_fallback": q.is_fallback, + } + ) if dry_run: continue series = client.query_range( @@ -788,8 +793,7 @@ def correlate( peak_aggregate=peak_value, sol_pct=None, notes=[ - f"counter-fallback metric {q.metric}: byte-rate " - "derivation not implemented; refer to raw payload" + f"counter-fallback metric {q.metric}: byte-rate derivation not implemented; refer to raw payload" ], ) @@ -810,13 +814,15 @@ def correlate( power_watts_per_gpu, power_promql = _query_mean_power_watts_per_gpu( client, inputs, step_s=float(expected_scrape_s or 15) ) - query_records.append({ - "peak_key": "power_watts_per_gpu", - "metric": _POWER_METRIC, - "promql": power_promql, - "unit": "watts", - "is_fallback": False, - }) + query_records.append( + { + "peak_key": "power_watts_per_gpu", + "metric": _POWER_METRIC, + "promql": power_promql, + "unit": "watts", + "is_fallback": False, + } + ) result = DcgmCorrelationResult( schema_version=1, @@ -849,12 +855,10 @@ def correlate( except json.JSONDecodeError: kernels_payload = None if kernels_payload is not None: - result.per_category_attribution = cross_attribute_zymtrace( - result, kernels_payload, ceilings - ) + result.per_category_attribution = cross_attribute_zymtrace(result, kernels_payload, ceilings) result.kernels_json_path = str(kernels_json_path) if "zymtrace" not in result.captured_sources: - result.captured_sources = list(result.captured_sources) + ["zymtrace"] + result.captured_sources = [*list(result.captured_sources), "zymtrace"] return result @@ -1012,8 +1016,7 @@ def _resource_from_frozen( notes=notes, ) raise ValueError( - f"_resource_from_frozen: unsupported unit {unit!r} on peak_key={peak_key} " - f"(expected 'ratio' or 'bytes_per_s')" + f"_resource_from_frozen: unsupported unit {unit!r} on peak_key={peak_key} (expected 'ratio' or 'bytes_per_s')" ) @@ -1061,9 +1064,7 @@ def correlate_from_frozen( for required in ("hw_key", "n_gpus", "sweep_start_utc", "sweep_end_utc", "resources"): if required not in data: - raise FrozenYamlMalformed( - frozen_yaml, reason=f"missing required key '{required}'" - ) + raise FrozenYamlMalformed(frozen_yaml, reason=f"missing required key '{required}'") hw_key = data["hw_key"] if hw_key not in ceilings: @@ -1073,33 +1074,23 @@ def correlate_from_frozen( ) hw_data = ceilings[hw_key] - sweep_start = datetime.fromisoformat( - str(data["sweep_start_utc"]).replace("Z", "+00:00") - ).astimezone(timezone.utc) - sweep_end = datetime.fromisoformat( - str(data["sweep_end_utc"]).replace("Z", "+00:00") - ).astimezone(timezone.utc) + sweep_start = datetime.fromisoformat(str(data["sweep_start_utc"]).replace("Z", "+00:00")).astimezone(UTC) + sweep_end = datetime.fromisoformat(str(data["sweep_end_utc"]).replace("Z", "+00:00")).astimezone(UTC) duration_s = (sweep_end - sweep_start).total_seconds() if duration_s <= 0: - raise FrozenYamlMalformed( - frozen_yaml, reason=f"non-positive duration_s={duration_s}" - ) + raise FrozenYamlMalformed(frozen_yaml, reason=f"non-positive duration_s={duration_s}") n_gpus = int(data["n_gpus"]) resources_in = data["resources"] if not isinstance(resources_in, list) or not resources_in: - raise FrozenYamlMalformed( - frozen_yaml, reason="resources must be a non-empty list" - ) + raise FrozenYamlMalformed(frozen_yaml, reason="resources must be a non-empty list") resources: list[ResourceResult] = [] queries: list[dict[str, Any]] = [] for entry in resources_in: if not isinstance(entry, dict): - raise FrozenYamlMalformed( - frozen_yaml, reason=f"resource entry not a mapping: {entry!r}" - ) + raise FrozenYamlMalformed(frozen_yaml, reason=f"resource entry not a mapping: {entry!r}") for required in ("peak_key", "metric", "unit", "measured_avg"): if required not in entry: raise FrozenYamlMalformed( @@ -1107,39 +1098,31 @@ def correlate_from_frozen( reason=f"resource entry missing '{required}': {entry!r}", ) resources.append(_resource_from_frozen(entry, n_gpus, duration_s, hw_data)) - queries.append({ - "peak_key": entry["peak_key"], - "metric": entry["metric"], - "promql": str(entry.get("promql", "")), - "unit": entry["unit"], - "is_fallback": False, - }) + queries.append( + { + "peak_key": entry["peak_key"], + "metric": entry["metric"], + "promql": str(entry.get("promql", "")), + "unit": entry["unit"], + "is_fallback": False, + } + ) # Optional mean per-GPU power (watts) from the frozen YAML -> tokens-per-watt. power_in = data.get("power_watts_per_gpu") - power_watts_per_gpu = ( - float(power_in) if isinstance(power_in, (int, float)) and power_in > 0 else None - ) + power_watts_per_gpu = float(power_in) if isinstance(power_in, (int, float)) and power_in > 0 else None # Optional re-query provenance: node(s) + pod scope. nodes_in = data.get("nodes") - nodes = ( - [str(n).strip() for n in nodes_in if str(n).strip()] - if isinstance(nodes_in, list) - else [] - ) + nodes = [str(n).strip() for n in nodes_in if str(n).strip()] if isinstance(nodes_in, list) else [] namespace_in = data.get("namespace") namespace = str(namespace_in) if isinstance(namespace_in, str) and namespace_in.strip() else None selector_in = data.get("pod_label_selector") - pod_label_selector = ( - str(selector_in) if isinstance(selector_in, str) and selector_in.strip() else None - ) + pod_label_selector = str(selector_in) if isinstance(selector_in, str) and selector_in.strip() else None result = DcgmCorrelationResult( schema_version=1, - captured_sources=["dcgm"] + ( - ["zymtrace"] if (cell_dir or kernels_json_path) else [] - ), + captured_sources=["dcgm"] + (["zymtrace"] if (cell_dir or kernels_json_path) else []), hw_key=hw_key, sweep_start_utc=sweep_start.strftime("%Y-%m-%dT%H:%M:%SZ"), sweep_end_utc=sweep_end.strftime("%Y-%m-%dT%H:%M:%SZ"), @@ -1165,9 +1148,7 @@ def correlate_from_frozen( if kernels_json_path is not None and kernels_json_path.is_file(): kernels_payload = json.loads(kernels_json_path.read_text()) - result.per_category_attribution = cross_attribute_zymtrace( - result, kernels_payload, ceilings - ) + result.per_category_attribution = cross_attribute_zymtrace(result, kernels_payload, ceilings) result.kernels_json_path = str(kernels_json_path) return result @@ -1196,10 +1177,10 @@ def read_sweep_window_from_bundle(bundle_path: Path) -> tuple[datetime, datetime duration_s = float(ipb["bench"]["duration_effective_s"]) # Parse ISO-8601; tolerate trailing Z. cs = captured_at.replace("Z", "+00:00") - start = datetime.fromisoformat(cs).astimezone(timezone.utc) + start = datetime.fromisoformat(cs).astimezone(UTC) # Production captured_at may be either start or end; the rest of the # codebase uses it as a timestamp at capture write-time = effectively # end-of-sweep. Convention here: treat captured_at as start; user # who knows otherwise can pass explicit start/end to DcgmCorrelateInputs. - end = datetime.fromtimestamp(start.timestamp() + duration_s, tz=timezone.utc) + end = datetime.fromtimestamp(start.timestamp() + duration_s, tz=UTC) return start, end diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/experiments_index.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/experiments_index.py index ab34096..7cdfaa2 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/experiments_index.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/experiments_index.py @@ -18,8 +18,9 @@ import json import re +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any from tools.perf_tune_report.lake_writer import ( parse_campaign_utc, @@ -42,8 +43,8 @@ def enumerate_published_campaign_ids( Hive ``campaign=`` token from each key. Isolated + factory-injectable so tests stub the S3 client (mirrors ``upload_to_s3``).""" from tools.perf_tune_report.lake_writer import ( - S3_PREFIX, CAMPAIGN_TABLE_NAME, + S3_PREFIX, _make_s3_client, ) @@ -133,11 +134,7 @@ def build_index_row(campaign_dir: Path, published: set[str] | None = None) -> di def build_index(campaigns_root: Path, published: set[str] | None = None) -> list[dict[str, Any]]: """Enumerate every local campaign into index rows, sorted by captured_at.""" - rows = [ - build_index_row(d, published) - for d in sorted(campaigns_root.iterdir()) - if _is_campaign_dir(d) - ] + rows = [build_index_row(d, published) for d in sorted(campaigns_root.iterdir()) if _is_campaign_dir(d)] rows.sort(key=lambda r: (r["captured_at_utc"], r["campaign_id"])) return rows @@ -265,8 +262,7 @@ def build_inventory( "bundle_only_count": len(bundle_only), "published_in_lake": (len(published) if published is not None else None), "published_local_campaigns": ( - sum(1 for r in campaigns if r.get("published_to_lake")) - if published is not None else None + sum(1 for r in campaigns if r.get("published_to_lake")) if published is not None else None ), "by_family": dict(sorted(by_family.items(), key=lambda kv: (-kv[1], kv[0]))), "by_model": dict(sorted(by_model.items(), key=lambda kv: (-kv[1], kv[0]))), @@ -292,7 +288,7 @@ def render_inventory_md(inv: dict[str, Any]) -> str: "", "Generated by `perftunereport experiment_inventory`; do not hand-edit. The run-id is the " "join key across evidence bundle, cluster `experiment=` label, and perf-lake " - "campaign (AGENTS.md \"Experiment isolation and traceability\"). Pass `--bundle-root` " + 'campaign (AGENTS.md "Experiment isolation and traceability"). Pass `--bundle-root` ' "(repeatable) to include the deploy-bundle trees; with none given the count is " "campaigns-only.", "", diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/fixtures/_build_synthetic_atlas.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/fixtures/_build_synthetic_atlas.py index 7fed1ae..b0f7498 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/fixtures/_build_synthetic_atlas.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/fixtures/_build_synthetic_atlas.py @@ -25,7 +25,6 @@ import json import sys from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[3] @@ -160,10 +159,10 @@ def _scale_anchor(tps_anchor: float, ttft_anchor: float, c_anchor: int, c_target ratio = c_target / c_anchor # tps/GPU rises with concurrency up to saturation. Below C=8 use ratio**0.85 # to approximate the diminishing-returns shape vLLM produces. - tps = tps_anchor * (ratio ** 0.85) + tps = tps_anchor * (ratio**0.85) # TTFT below C=8: divide by ratio**0.6 so going from C=8 to C=1 cuts # TTFT by ~3.8x. Realistic for prefill-dominated workloads. - ttft = ttft_anchor * (ratio ** 0.6) if ratio < 1.0 else ttft_anchor + ttft = ttft_anchor * (ratio**0.6) if ratio < 1.0 else ttft_anchor return tps, ttft @@ -244,13 +243,9 @@ def build_rows() -> list[AtlasCell]: tps_per_gpu = float(raw_tps) ttft_ms = float(raw_ttft) else: - tps_per_gpu, ttft_ms = _scale_anchor( - float(c8_tps), float(C8_TTFT_MS[v_idx][m_idx]), 8, c - ) + tps_per_gpu, ttft_ms = _scale_anchor(float(c8_tps), float(C8_TTFT_MS[v_idx][m_idx]), 8, c) - req_throughput, tps_per_user = _derived_metrics( - tps_per_gpu, c, variant.tp - ) + req_throughput, tps_per_user = _derived_metrics(tps_per_gpu, c, variant.tp) rows.append( AtlasCell( @@ -278,9 +273,7 @@ def build_rows() -> list[AtlasCell]: for hw, tp in (("B200", 8), ("GB300", 4)): for strategy in ("EP", "TP"): for mbt in MBT_VALUES: - cell_id = ( - f"{hw.lower()}-nvfp4-tp{tp}-{strategy.lower()}-mtp-mbt{mbt}" - ) + cell_id = f"{hw.lower()}-nvfp4-tp{tp}-{strategy.lower()}-mtp-mbt{mbt}" rows.append( AtlasCell( cell_id=cell_id, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/fleet_leaderboard.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/fleet_leaderboard.py index 9a138f6..23972b6 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/fleet_leaderboard.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/fleet_leaderboard.py @@ -19,14 +19,18 @@ from __future__ import annotations +import logging import re from pathlib import Path from typing import Any +import yaml + from tools.perf_tune_report.schema import AtlasCell, read_jsonl GPU_HR_DEFAULT = 8.60 # GB300 $/gpu-hour SHAPES = ("aa-1k", "aa-10k", "aa-100k") +logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # @@ -44,8 +48,10 @@ def canon_model(raw: str | None) -> str: m = re.sub(r"-(it|Instruct)$", "", m, flags=re.IGNORECASE) m = re.sub(r"-2507$", "", m) table = { - "gemma-4-26B-A4B": "Gemma-4-26B-A4B", "Gemma-4-26B-A4B": "Gemma-4-26B-A4B", - "Llama-3.1-8B": "Llama-3.1-8B", "Llama-3.3-70B": "Llama-3.3-70B", + "gemma-4-26B-A4B": "Gemma-4-26B-A4B", + "Gemma-4-26B-A4B": "Gemma-4-26B-A4B", + "Llama-3.1-8B": "Llama-3.1-8B", + "Llama-3.3-70B": "Llama-3.3-70B", } return table.get(m, m) @@ -66,7 +72,8 @@ def read_all_rows(campaigns_root: Path, hardware_filter: str = "GB300") -> list[ for r in read_jsonl(atlas): if hardware_filter in (r.hardware or ""): rows.append(r) - except Exception: # noqa: BLE001 - skip a malformed campaign, never fail the fleet view + except (OSError, TypeError, ValueError) as exc: + logger.debug("Skipping malformed campaign atlas %s: %s", atlas, exc) continue return rows @@ -75,8 +82,7 @@ def _cost(tps: float | None, gpu_hr: float) -> float | None: return (gpu_hr / 3600.0) / tps * 1e6 if isinstance(tps, (int, float)) and tps else None -def resolve_gpu_hr(hardware: str, configs_dir: Path | None = None, - override: float | None = None) -> float: +def resolve_gpu_hr(hardware: str, configs_dir: Path | None = None, override: float | None = None) -> float: """Resolve the $/GPU-hour rate (the adjustable cost knob). An explicit --gpu-hr override wins; else read ``perf-tune-report/configs/cost.yaml`` ``usd_per_gpu_hour[hardware]`` (or ``[default]``); else ``GPU_HR_DEFAULT``. Cost is a simple lookup, not a model.""" @@ -86,14 +92,18 @@ def resolve_gpu_hr(hardware: str, configs_dir: Path | None = None, cost_yaml = Path(configs_dir) / "cost.yaml" if cost_yaml.is_file(): try: - import yaml # type: ignore - rates = (yaml.safe_load(cost_yaml.read_text()) or {}).get("usd_per_gpu_hour") or {} + data = yaml.safe_load(cost_yaml.read_text()) or {} + if not isinstance(data, dict): + return GPU_HR_DEFAULT + rates = data.get("usd_per_gpu_hour") or {} + if not isinstance(rates, dict): + return GPU_HR_DEFAULT if hardware in rates: return float(rates[hardware]) if "default" in rates: return float(rates["default"]) - except Exception: # noqa: BLE001 - cost is best-effort; fall back to the default - pass + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Ignoring malformed cost config %s: %s", cost_yaml, exc) return GPU_HR_DEFAULT @@ -118,7 +128,9 @@ def build_aa(rows: list[AtlasCell]) -> dict[str, dict[tuple[str, int], dict[str, if seen_at.get((model, key), "") <= cap: seen_at[(model, key)] = cap entry = { - "ttft": r.ttft_avg_ms, "opu": r.output_tps_per_user, "opg": r.output_tps_per_gpu, + "ttft": r.ttft_avg_ms, + "opu": r.output_tps_per_user, + "opg": r.output_tps_per_gpu, "ttfo": getattr(r, "ttfo_avg_ms", None), "cov": getattr(r, "ttfo_coverage", None), "rtoks": getattr(r, "reasoning_token_count", None), @@ -128,9 +140,13 @@ def build_aa(rows: list[AtlasCell]) -> dict[str, dict[tuple[str, int], dict[str, entry[k] = prev[k] data.setdefault(model, {})[key] = entry elif prev is not None and prev.get("rtoks") is None and getattr(r, "reasoning_token_count", None) is not None: - prev.update({"ttfo": getattr(r, "ttfo_avg_ms", None), - "cov": getattr(r, "ttfo_coverage", None), - "rtoks": getattr(r, "reasoning_token_count", None)}) + prev.update( + { + "ttfo": getattr(r, "ttfo_avg_ms", None), + "cov": getattr(r, "ttfo_coverage", None), + "rtoks": getattr(r, "reasoning_token_count", None), + } + ) return data @@ -152,9 +168,14 @@ def build_throughput(rows: list[AtlasCell]) -> list[dict[str, Any]]: key = (canon_model(r.model), str(r.quant or "").upper(), tp) if key not in best or tps > best[key]["tps_gpu"]: best[key] = { - "model": key[0], "quant": key[1], "tp": tp, "tps_gpu": tps, - "conc": r.concurrency, "tps_user": r.output_tps_per_user, - "ttft": r.ttft_avg_ms, "mnbt": r.max_num_batched_tokens, + "model": key[0], + "quant": key[1], + "tp": tp, + "tps_gpu": tps, + "conc": r.concurrency, + "tps_user": r.output_tps_per_user, + "ttft": r.ttft_avg_ms, + "mnbt": r.max_num_batched_tokens, } return list(best.values()) @@ -226,30 +247,48 @@ def _f(x: Any, n: int = 1) -> str: def render_aa_md(data: dict, hw: str, gpu_hr: float) -> str: - L = [f"# Artificial Analysis (AA) fleet leaderboard - {hw} TP4 NVFP4", "", - f"AUTO-GENERATED by `perftunereport fleet_leaderboard` ({len(data)} models). Re-run after any new AA " - "campaign publishes. Same methodology (AIPerf synthetic, temp 0 / top_p 1 / min_tokens+ignore_eos, " - f"o200k_base), same HW ({hw}, TP=4, NVFP4). Read-only synthesis.", "", - "## Latency tier (c=1, single prompt) - sorted by output tok/s/user"] + L = [ + f"# Artificial Analysis (AA) fleet leaderboard - {hw} TP4 NVFP4", + "", + f"AUTO-GENERATED by `perftunereport fleet_leaderboard` ({len(data)} models). Re-run after any new AA " + "campaign publishes. Same methodology (AIPerf synthetic, temp 0 / top_p 1 / min_tokens+ignore_eos, " + f"o200k_base), same HW ({hw}, TP=4, NVFP4). Read-only synthesis.", + "", + "## Latency tier (c=1, single prompt) - sorted by output tok/s/user", + ] for s in SHAPES: - ranked = sorted([(m, data[m][(s, 1)]) for m in data if (s, 1) in data[m]], - key=lambda r: -(r[1].get("opu") or 0)) + ranked = sorted( + [(m, data[m][(s, 1)]) for m in data if (s, 1) in data[m]], key=lambda r: -(r[1].get("opu") or 0) + ) if not ranked: continue - L += ["", f"### {s} (c=1)", "", "| rank | model | TTFT (ms) | output tok/s/user | tok/s/GPU |", - "| ---: | --- | ---: | ---: | ---: |"] + L += [ + "", + f"### {s} (c=1)", + "", + "| rank | model | TTFT (ms) | output tok/s/user | tok/s/GPU |", + "| ---: | --- | ---: | ---: | ---: |", + ] for i, (m, v) in enumerate(ranked, 1): L.append(f"| {i} | {m} | {_f(v['ttft'])} | {_f(v['opu'])} | {_f(v['opg'])} |") - L += ["", "## Answer latency (TTFO, reasoning-aware; c=1) - TTFO is an ANSWERED-SUBSET stat, always read with its coverage"] + L += [ + "", + "## Answer latency (TTFO, reasoning-aware; c=1) - TTFO is an ANSWERED-SUBSET stat, always read with its coverage", + ] for s in SHAPES: - ranked = sorted([(m, data[m][(s, 1)]) for m in data - if (s, 1) in data[m] and data[m][(s, 1)].get("cov") is not None], - key=lambda r: (r[1].get("ttfo") is None, r[1].get("ttfo") or 0)) + ranked = sorted( + [(m, data[m][(s, 1)]) for m in data if (s, 1) in data[m] and data[m][(s, 1)].get("cov") is not None], + key=lambda r: (r[1].get("ttfo") is None, r[1].get("ttfo") or 0), + ) if not ranked: continue - L += ["", f"### {s} (c=1)", "", - "| rank | model | TTFO (ms) | coverage | think toks (all reqs) | TTFT (ms) |", - "| ---: | --- | ---: | ---: | ---: | ---: |"] + L += [ + "", + f"### {s} (c=1)", + "", + "| rank | model | TTFO (ms) | coverage | think toks (all reqs) | TTFT (ms) |", + "| ---: | --- | ---: | ---: | ---: | ---: |", + ] for i, (m, v) in enumerate(ranked, 1): cov = f"{v['cov']:.0%}" if v.get("cov") is not None else "NA" ttfo = _f(v["ttfo"]) if v.get("ttfo") is not None else "never answers" @@ -257,63 +296,98 @@ def render_aa_md(data: dict, hw: str, gpu_hr: float) -> str: L.append(f"| {i} | {m} | {ttfo} | {cov} | {rt} | {_f(v['ttft'])} |") L += ["", "## Parallel tier (c=10) - sorted by tok/s/GPU"] for s in ("aa-1k", "aa-10k"): - ranked = sorted([(m, data[m][(s, 10)]) for m in data if (s, 10) in data[m]], - key=lambda r: -(r[1].get("opg") or 0)) + ranked = sorted( + [(m, data[m][(s, 10)]) for m in data if (s, 10) in data[m]], key=lambda r: -(r[1].get("opg") or 0) + ) if not ranked: continue - L += ["", f"### {s} (c=10)", "", "| rank | model | TTFT (ms) | tok/s/user | tok/s/GPU |", - "| ---: | --- | ---: | ---: | ---: |"] + L += [ + "", + f"### {s} (c=10)", + "", + "| rank | model | TTFT (ms) | tok/s/user | tok/s/GPU |", + "| ---: | --- | ---: | ---: | ---: |", + ] for i, (m, v) in enumerate(ranked, 1): L.append(f"| {i} | {m} | {_f(v['ttft'])} | {_f(v['opu'])} | {_f(v['opg'])} |") - L += ["", "## Read", - "- Active-param count drives decode speed (per-token HBM traffic), but absolute GPU utilization is " - "LOW (HBM <15%, L3-grounded) -> host/KV-scheduling-bound, NOT memory-bound. Grounded cross-model " - "attribution: `FLEET-THROUGHPUT-ATTRIBUTION-GB300.md`.", - "- MiniMax-M2.7 typically has the lowest TTFT; GLM-5.1 (DSA sparse) is the long-context outlier.", "", - f"## Cost ($/1M output tok @ ${gpu_hr}/GPU-hour) - cheapest first", "", - "| model | aa-1k c=1 | aa-1k c=10 | aa-10k c=1 |", "| --- | ---: | ---: | ---: |"] + L += [ + "", + "## Read", + "- Active-param count drives decode speed (per-token HBM traffic), but absolute GPU utilization is " + "LOW (HBM <15%, L3-grounded) -> host/KV-scheduling-bound, NOT memory-bound. Grounded cross-model " + "attribution: `FLEET-THROUGHPUT-ATTRIBUTION-GB300.md`.", + "- MiniMax-M2.7 typically has the lowest TTFT; GLM-5.1 (DSA sparse) is the long-context outlier.", + "", + f"## Cost ($/1M output tok @ ${gpu_hr}/GPU-hour) - cheapest first", + "", + "| model | aa-1k c=1 | aa-1k c=10 | aa-10k c=1 |", + "| --- | ---: | ---: | ---: |", + ] crows = [] for m in data: c1 = _cost(data[m].get(("aa-1k", 1), {}).get("opg"), gpu_hr) c10 = _cost(data[m].get(("aa-1k", 10), {}).get("opg"), gpu_hr) c10k = _cost(data[m].get(("aa-10k", 1), {}).get("opg"), gpu_hr) crows.append((m, c1, c10, c10k)) - crows.sort(key=lambda r: (r[1] if r[1] is not None else 9e9)) - dol = lambda x: ("$%.2f" % x) if isinstance(x, (int, float)) else "(DNF)" + crows.sort(key=lambda r: r[1] if r[1] is not None else 9e9) + + def dol(value: float | None) -> str: + return f"${value:.2f}" if isinstance(value, (int, float)) else "(DNF)" + for m, c1, c10, c10k in crows: L.append(f"| {m} | {dol(c1)} | {dol(c10)} | {dol(c10k)} |") - L += ["", "Batching (c=10) is ~7x cheaper than single-prompt (c=1). Companion: " - "`THROUGHPUT-FLEET-LEADERBOARD-GB300.md`, `FLEET-MODEL-SELECTION-GB300.md`.", ""] + L += [ + "", + "Batching (c=10) is ~7x cheaper than single-prompt (c=1). Companion: " + "`THROUGHPUT-FLEET-LEADERBOARD-GB300.md`, `FLEET-MODEL-SELECTION-GB300.md`.", + "", + ] return "\n".join(L) def render_throughput_md(allrows: list[dict], hw: str, gpu_hr: float) -> str: def table(rs: list[dict], header: str) -> list[str]: rs = sorted(rs, key=lambda r: -r["tps_gpu"]) - out = [header, "", "| rank | model | quant | TP | peak tok/s/GPU | @conc | tok/s/user@peak | " - "TTFT@peak (ms) | $/1M out |", "| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |"] + out = [ + header, + "", + "| rank | model | quant | TP | peak tok/s/GPU | @conc | tok/s/user@peak | TTFT@peak (ms) | $/1M out |", + "| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] for i, r in enumerate(rs, 1): c = _cost(r["tps_gpu"], gpu_hr) - out.append(f"| {i} | {r['model']} | {r['quant']} | {r['tp']} | {_f(r['tps_gpu'])} | {r['conc']} | " - f"{_f(r['tps_user'])} | {_f(r['ttft'])} | {('$%.2f' % c) if c else 'NA'} |") + out.append( + f"| {i} | {r['model']} | {r['quant']} | {r['tp']} | {_f(r['tps_gpu'])} | {r['conc']} | " + f"{_f(r['tps_user'])} | {_f(r['ttft'])} | {f'${c:.2f}' if c else 'NA'} |" + ) return out + tp4 = [r for r in allrows if r["tp"] == 4] alt = [r for r in allrows if r["tp"] in (2, 8)] nmodels = len({r["model"] for r in tp4}) - L = [f"# Throughput fleet leaderboard (peak tok/s/GPU) - {hw} NVFP4", "", - "> **PROVISIONAL** when generated mid-sweep (the roofline sweep is an active workstream; numbers are " - "best-so-far). Re-run `perftunereport fleet_leaderboard` to refresh.", "", - f"AUTO-GENERATED ({nmodels} models, TP=4). Peak = max sustained output tok/s/GPU across each config's " - f"concurrency sweep, with the latency AT that peak. {hw} / status=full only; AA cells excluded " - "(latency leaderboard). TP=1 microbenches excluded; TP=2/8 = sharding study.", ""] + L = [ + f"# Throughput fleet leaderboard (peak tok/s/GPU) - {hw} NVFP4", + "", + "> **PROVISIONAL** when generated mid-sweep (the roofline sweep is an active workstream; numbers are " + "best-so-far). Re-run `perftunereport fleet_leaderboard` to refresh.", + "", + f"AUTO-GENERATED ({nmodels} models, TP=4). Peak = max sustained output tok/s/GPU across each config's " + f"concurrency sweep, with the latency AT that peak. {hw} / status=full only; AA cells excluded " + "(latency leaderboard). TP=1 microbenches excluded; TP=2/8 = sharding study.", + "", + ] L += table(tp4, "## Production tier (TP=4) - sorted by peak tok/s/GPU") if alt: L += ["", *table(alt, "## Alternate-sharding study (TP=2 / TP=8)")] - L += ["", "## Read", - "- Peak throughput is a batch-decode number (high concurrency) - the inverse-latency end of the " - "Pareto front. DCGM-grounded: even at peak the small-active MoEs sit at HBM <8% / tensor <3% " - "(GPU-starved, host/KV-bound) - see `FLEET-THROUGHPUT-ATTRIBUTION-GB300.md`.", - "- $/1M here is the throughput-optimal cost (~7x cheaper than the AA c=1 latency-optimal cost).", ""] + L += [ + "", + "## Read", + "- Peak throughput is a batch-decode number (high concurrency) - the inverse-latency end of the " + "Pareto front. DCGM-grounded: even at peak the small-active MoEs sit at HBM <8% / tensor <3% " + "(GPU-starved, host/KV-bound) - see `FLEET-THROUGHPUT-ATTRIBUTION-GB300.md`.", + "- $/1M here is the throughput-optimal cost (~7x cheaper than the AA c=1 latency-optimal cost).", + "", + ] return "\n".join(L) @@ -324,64 +398,92 @@ def render_pareto_md(M: dict, hw: str, gpu_hr: float, quality: dict | None = Non best_decode = max(names, key=lambda n: M[n]["lat_user"]) if names else None best_ttft = min(names, key=lambda n: M[n]["lat_ttft"]) if names else None best_tput = max(names, key=lambda n: M[n]["tps_gpu"]) if names else None - L = [f"# Fleet model-selection guide (perf Pareto frontier) - {hw} TP4 NVFP4", "", - "> **PROVISIONAL** where the throughput axis is still filling. Capstone over the AA latency + " - "throughput leaderboards. Re-run `perftunereport fleet_leaderboard`.", "", - f"AUTO-GENERATED ({len(names)} models with all 3 axes). Axes: decode latency (aa-1k c=1 tok/s/user, " - "higher better), first-token (aa-1k c=1 TTFT, lower better), batch throughput (peak tok/s/GPU TP=4, " - "higher better).", ""] + L = [ + f"# Fleet model-selection guide (perf Pareto frontier) - {hw} TP4 NVFP4", + "", + "> **PROVISIONAL** where the throughput axis is still filling. Capstone over the AA latency + " + "throughput leaderboards. Re-run `perftunereport fleet_leaderboard`.", + "", + f"AUTO-GENERATED ({len(names)} models with all 3 axes). Axes: decode latency (aa-1k c=1 tok/s/user, " + "higher better), first-token (aa-1k c=1 TTFT, lower better), batch throughput (peak tok/s/GPU TP=4, " + "higher better).", + "", + ] if names: - L += ["## Pick by priority", "", "| optimize for... | pick | number |", "| --- | --- | --- |", - f"| interactive decode | **{best_decode}** | {_f(M[best_decode]['lat_user'])} tok/s/user |", - f"| lowest first-token | **{best_ttft}** | {_f(M[best_ttft]['lat_ttft'])} ms TTFT |", - f"| batch throughput / $ | **{best_tput}** | {_f(M[best_tput]['tps_gpu'])} tok/s/GPU " - f"(${_cost(M[best_tput]['tps_gpu'], gpu_hr):.2f}/1M) |", ""] - L += ["## Perf-efficient frontier (non-dominated - choose from these)", "", - "| model | decode tok/s/user | TTFT (ms) | peak tok/s/GPU | $/1M out |", - "| --- | ---: | ---: | ---: | ---: |"] + L += [ + "## Pick by priority", + "", + "| optimize for... | pick | number |", + "| --- | --- | --- |", + f"| interactive decode | **{best_decode}** | {_f(M[best_decode]['lat_user'])} tok/s/user |", + f"| lowest first-token | **{best_ttft}** | {_f(M[best_ttft]['lat_ttft'])} ms TTFT |", + f"| batch throughput / $ | **{best_tput}** | {_f(M[best_tput]['tps_gpu'])} tok/s/GPU " + f"(${_cost(M[best_tput]['tps_gpu'], gpu_hr):.2f}/1M) |", + "", + ] + L += [ + "## Perf-efficient frontier (non-dominated - choose from these)", + "", + "| model | decode tok/s/user | TTFT (ms) | peak tok/s/GPU | $/1M out |", + "| --- | ---: | ---: | ---: | ---: |", + ] for n in sorted(frontier, key=lambda x: -M[x]["tps_gpu"]): e = M[n] - L.append(f"| {n} | {_f(e['lat_user'])} | {_f(e['lat_ttft'])} | {_f(e['tps_gpu'])} | " - f"${_cost(e['tps_gpu'], gpu_hr):.2f} |") + L.append( + f"| {n} | {_f(e['lat_user'])} | {_f(e['lat_ttft'])} | {_f(e['tps_gpu'])} | " + f"${_cost(e['tps_gpu'], gpu_hr):.2f} |" + ) L += ["", "## Perf-dominated (beaten on ALL three axes)", ""] if dominated: - L += ["| model | decode tok/s/user | TTFT (ms) | peak tok/s/GPU | dominated by |", - "| --- | ---: | ---: | ---: | --- |"] + L += [ + "| model | decode tok/s/user | TTFT (ms) | peak tok/s/GPU | dominated by |", + "| --- | ---: | ---: | ---: | --- |", + ] for n in sorted(dominated, key=lambda x: -M[x]["tps_gpu"]): e = M[n] dom = [o for o in frontier if dominates(M[o], e)] - L.append(f"| {n} | {_f(e['lat_user'])} | {_f(e['lat_ttft'])} | {_f(e['tps_gpu'])} | " - f"{', '.join(dom) or '-'} |") + L.append( + f"| {n} | {_f(e['lat_user'])} | {_f(e['lat_ttft'])} | {_f(e['tps_gpu'])} | {', '.join(dom) or '-'} |" + ) else: L.append("_None - every measured model is on the perf frontier._") - L += ["", "## Model quality (lm-eval serving evals, where measured)", - "Quality is the REAL model-selection driver, shown SEPARATELY from the perf Pareto " - "(a perf-dominated model may be the right pick on quality). Captured via " - "`perftunereport import_model_eval` -> quality_v1; empty until serving evals are imported.", - ""] + L += [ + "", + "## Model quality (lm-eval serving evals, where measured)", + "Quality is the REAL model-selection driver, shown SEPARATELY from the perf Pareto " + "(a perf-dominated model may be the right pick on quality). Captured via " + "`perftunereport import_model_eval` -> quality_v1; empty until serving evals are imported.", + "", + ] measured = {m: q for m, q in (quality or {}).items() if q} if measured: metric_names = sorted({n for q in measured.values() for n in q}) - L += ["| model | " + " | ".join(metric_names) + " |", - "| --- | " + " | ".join(["---:"] * len(metric_names)) + " |"] + L += [ + "| model | " + " | ".join(metric_names) + " |", + "| --- | " + " | ".join(["---:"] * len(metric_names)) + " |", + ] for m in sorted(measured): cells = " | ".join( - (_f(measured[m].get(n), 3) if measured[m].get(n) is not None else "-") - for n in metric_names + (_f(measured[m].get(n), 3) if measured[m].get(n) is not None else "-") for n in metric_names ) L.append(f"| {m} | {cells} |") else: L.append("_None measured yet -- run `inference-model-eval` + `perftunereport import_model_eval`._") - L += ["", "## Critical caveat: PERF-only Pareto, NOT model quality", - "- The Pareto frontier above ranks PERF only; \"dominated\" = perf-dominated on the AA " - "short-context shape, NOT \"never use\". Cross-reference the quality table above (reasoning, " - "long-context, agentic quality is orthogonal and is usually the real selection driver). " - "GLM-5.1's value is DSA-sparse long-context, not the AA 1k-shape decode this frontier measures.", ""] + L += [ + "", + "## Critical caveat: PERF-only Pareto, NOT model quality", + '- The Pareto frontier above ranks PERF only; "dominated" = perf-dominated on the AA ' + 'short-context shape, NOT "never use". Cross-reference the quality table above (reasoning, ' + "long-context, agentic quality is orthogonal and is usually the real selection driver). " + "GLM-5.1's value is DSA-sparse long-context, not the AA 1k-shape decode this frontier measures.", + "", + ] return "\n".join(L) -def write_leaderboards(rows: list[AtlasCell], out_dir: Path, hw: str = "GB300", - gpu_hr: float = GPU_HR_DEFAULT) -> dict[str, str]: +def write_leaderboards( + rows: list[AtlasCell], out_dir: Path, hw: str = "GB300", gpu_hr: float = GPU_HR_DEFAULT +) -> dict[str, int | str]: """Render + write all three leaderboards; return {name: path}.""" out_dir.mkdir(parents=True, exist_ok=True) aa = build_aa(rows) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/graph_diff/graph_diff.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/graph_diff/graph_diff.py index 051ae77..c7aef73 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/graph_diff/graph_diff.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/graph_diff/graph_diff.py @@ -67,12 +67,11 @@ import difflib import json import re -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from dataclasses import asdict, dataclass +from datetime import UTC, datetime from pathlib import Path from typing import Any - # --------------------------------------------------------------------------- # Log-parsing regexes # --------------------------------------------------------------------------- @@ -240,7 +239,8 @@ def _diff_one_graph( b_lines = side_b_text.splitlines() diff_text = "\n".join( difflib.unified_diff( - a_lines, b_lines, + a_lines, + b_lines, fromfile=str(a_path.name), tofile=str(b_path.name), lineterm="", @@ -307,7 +307,7 @@ def diff_graph_logs( raise ValueError(f"graph_diff: side-B log does not exist: {side_b_log}") if captured_at is None: - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") a_summary, a_graphs = _summarize_side(side_a_label, side_a_log) b_summary, b_graphs = _summarize_side(side_b_label, side_b_log) @@ -382,9 +382,7 @@ def diff_graph_logs( pass_summary = "; ".join(pass_summary_parts) else: pass_summary = "no compile-pass changes" - delta_label = ( - f"{delta_pct:+.1f}%" if delta_pct is not None else "n/a" - ) + delta_label = f"{delta_pct:+.1f}%" if delta_pct is not None else "n/a" summary = ( f"{n_pairs} graph(s) diffed; {pass_summary}; " f"node-count delta: {delta_label} " @@ -412,8 +410,6 @@ def diff_graph_logs( ) if not dry_run: - graph_diff_json_path.write_text( - json.dumps(result.to_dict(), indent=2, sort_keys=True) - ) + graph_diff_json_path.write_text(json.dumps(result.to_dict(), indent=2, sort_keys=True)) return result diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/helpers.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/helpers.py index 1a91257..8c09d57 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/helpers.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/helpers.py @@ -5,8 +5,7 @@ import json import os import re -import sys -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -103,7 +102,7 @@ def slugify(text: str) -> str: def utc_timestamp_slug() -> str: """``YYYYMMDDTHHMMSSZ`` per the workspace's evidence-bundle-init convention.""" - return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") def emit(payload: dict[str, Any], *, as_json: bool) -> None: diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/__init__.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/__init__.py index 0f3f437..a574e36 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/__init__.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/__init__.py @@ -39,34 +39,34 @@ from pathlib import Path from typing import Any -from tools.perf_tune_report.importers.inference_perf_bench import ( - ImportResult, - import_perf_bench_bundle, +from tools.perf_tune_report.importers.aiperf_export import ( + AiperfImportResult, + detect_aiperf_bundle, + import_aiperf_bundle, ) from tools.perf_tune_report.importers.inference_drive_load import ( DriveLoadImportResult, detect_bundle_pattern, import_drive_load_bundle, ) +from tools.perf_tune_report.importers.inference_perf_bench import ( + ImportResult, + import_perf_bench_bundle, +) from tools.perf_tune_report.importers.lws_summary import ( LwsSummaryImportResult, detect_lws_summary, import_lws_summary_bundle, ) -from tools.perf_tune_report.importers.aiperf_export import ( - AiperfImportResult, - detect_aiperf_bundle, - import_aiperf_bundle, +from tools.perf_tune_report.importers.roofline_sweep import ( + RooflineSweepImportResult, + import_roofline_sweep_bundle, ) from tools.perf_tune_report.importers.variant_ab import ( VariantAbImportResult, detect_variant_ab, import_variant_ab_bundle, ) -from tools.perf_tune_report.importers.roofline_sweep import ( - RooflineSweepImportResult, - import_roofline_sweep_bundle, -) def import_bundle_auto( @@ -78,7 +78,7 @@ def import_bundle_auto( captured_at: str | None = None, concurrency_override: int | None = None, require_plot_ready: bool = False, -) -> ImportResult | DriveLoadImportResult | LwsSummaryImportResult: +) -> ImportResult | DriveLoadImportResult | LwsSummaryImportResult | VariantAbImportResult | AiperfImportResult: """Detect the bundle pattern and dispatch to the right importer. Detection priority: lws_summary -> inference_perf_bench -> inference_drive_load. @@ -153,21 +153,21 @@ def import_bundle_auto( __all__ = [ - "ImportResult", + "AiperfImportResult", "DriveLoadImportResult", + "ImportResult", "LwsSummaryImportResult", - "AiperfImportResult", - "VariantAbImportResult", - "import_perf_bench_bundle", - "import_drive_load_bundle", - "import_lws_summary_bundle", - "import_aiperf_bundle", - "import_variant_ab_bundle", - "import_roofline_sweep_bundle", "RooflineSweepImportResult", - "import_bundle_auto", + "VariantAbImportResult", + "detect_aiperf_bundle", "detect_bundle_pattern", "detect_lws_summary", - "detect_aiperf_bundle", "detect_variant_ab", + "import_aiperf_bundle", + "import_bundle_auto", + "import_drive_load_bundle", + "import_lws_summary_bundle", + "import_perf_bench_bundle", + "import_roofline_sweep_bundle", + "import_variant_ab_bundle", ] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/aiperf_export.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/aiperf_export.py index 5f7c63b..7e3a497 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/aiperf_export.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/aiperf_export.py @@ -46,7 +46,7 @@ import csv import json from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -135,9 +135,7 @@ def _parse_aiperf_csv(path: Path) -> dict[str, float] | None: if "Time to First Token (ms)" in per_metric_avg: out["ttft_avg_ms"] = per_metric_avg["Time to First Token (ms)"] if "Output Token Throughput Per User (tokens/sec/user)" in per_metric_avg: - out["output_tps_per_user"] = per_metric_avg[ - "Output Token Throughput Per User (tokens/sec/user)" - ] + out["output_tps_per_user"] = per_metric_avg["Output Token Throughput Per User (tokens/sec/user)"] if "Inter Token Latency (ms)" in per_metric_avg: out["itl_avg_ms"] = per_metric_avg["Inter Token Latency (ms)"] if "Request Throughput (requests/sec)" in scalar: @@ -220,9 +218,7 @@ def import_aiperf_bundle( raise ValueError(f"import_aiperf: bundle does not exist: {bundle}") cells = _enumerate_aiperf_cells(bundle) if not cells: - raise ValueError( - f"import_aiperf: no c/profile_export_aiperf.csv under {bundle}" - ) + raise ValueError(f"import_aiperf: no c/profile_export_aiperf.csv under {bundle}") ov = overrides or {} model = ov.get("model") if not model: @@ -241,7 +237,7 @@ def import_aiperf_bundle( if isinstance(ov.get("expected_reqs"), dict): expected.update({int(k): int(v) for k, v in ov["expected_reqs"].items()}) if captured_at is None: - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") rows: list[AtlasCell] = [] partial_cells: list[int] = [] @@ -251,9 +247,7 @@ def import_aiperf_bundle( continue rc = m.get("request_count") exp = expected.get(cf.concurrency) - cell_partial = ( - rc is not None and exp is not None and rc < PARTIAL_FRACTION * exp - ) + cell_partial = rc is not None and exp is not None and rc < PARTIAL_FRACTION * exp status = STATUS_PARTIAL if cell_partial else STATUS_FULL if cell_partial: partial_cells.append(cf.concurrency) @@ -311,15 +305,18 @@ def import_aiperf_bundle( normalized_path = cell_dir / "normalized.json" if dry_run: return AiperfImportResult( - campaign_dir=campaign_dir, cell_id=cell_id, cell_dir=cell_dir, - normalized_path=normalized_path, bundle_path=bundle, - row_count=len(rows), concurrencies=concurrencies, - status=overall_status, partial_cells=sorted(partial_cells), + campaign_dir=campaign_dir, + cell_id=cell_id, + cell_dir=cell_dir, + normalized_path=normalized_path, + bundle_path=bundle, + row_count=len(rows), + concurrencies=concurrencies, + status=overall_status, + partial_cells=sorted(partial_cells), ) cell_dir.mkdir(parents=True, exist_ok=True) - normalized_path.write_text( - json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True) - ) + normalized_path.write_text(json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True)) (cell_dir / "status.txt").write_text(overall_status + "\n") (cell_dir / "backend.txt").write_text(BACKEND_AIPERF + "\n") (cell_dir / "SOURCE.md").write_text( @@ -332,8 +329,13 @@ def import_aiperf_bundle( f"- partial_cells (low completion): {sorted(partial_cells)}\n" ) return AiperfImportResult( - campaign_dir=campaign_dir, cell_id=cell_id, cell_dir=cell_dir, - normalized_path=normalized_path, bundle_path=bundle, - row_count=len(rows), concurrencies=concurrencies, - status=overall_status, partial_cells=sorted(partial_cells), + campaign_dir=campaign_dir, + cell_id=cell_id, + cell_dir=cell_dir, + normalized_path=normalized_path, + bundle_path=bundle, + row_count=len(rows), + concurrencies=concurrencies, + status=overall_status, + partial_cells=sorted(partial_cells), ) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_drive_load.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_drive_load.py index b46ef12..f6dda1c 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_drive_load.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_drive_load.py @@ -98,24 +98,26 @@ import re import statistics from dataclasses import asdict, dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any from tools.perf_tune_report.helpers import resolve_cell_dir - +from tools.perf_tune_report.importers.inference_perf_bench import ( + _CellIdentity, + _identity_from_meta_and_overrides, + _load_bundle_metadata, +) +from tools.perf_tune_report.importers.zymtrace_kernels import ( + KernelImportResult, + import_zymtrace_kernels, +) from tools.perf_tune_report.schema import ( BACKEND_VLLM_SWEEP, - STATUS_FAILED, STATUS_FULL, STATUS_PARTIAL, AtlasCell, ) -from tools.perf_tune_report.importers.zymtrace_kernels import ( - KernelImportResult, - import_zymtrace_kernels, -) - # Multi-concurrency layout: bundle/bench-c/raw/load.jsonl _BENCH_C_DIR = re.compile(r"^bench-c(\d+)$") @@ -129,9 +131,7 @@ class _JSONLFileInfo: concurrency: int -def _enumerate_jsonl_files( - bundle: Path, *, concurrency_override: int | None = None -) -> list[_JSONLFileInfo]: +def _enumerate_jsonl_files(bundle: Path, *, concurrency_override: int | None = None) -> list[_JSONLFileInfo]: """Find all load.jsonl files in the bundle. Two layouts supported: @@ -298,15 +298,6 @@ def to_dict(self) -> dict[str, Any]: return d -# Reuse identity helpers from the bench-serve importer (the metadata shape -# in inference_perfbench_v1.json is identical across the two importers). -from tools.perf_tune_report.importers.inference_perf_bench import ( - _CellIdentity, - _identity_from_meta_and_overrides, - _load_bundle_metadata, -) - - def _row_from_aggregate( identity: _CellIdentity, concurrency: int, @@ -316,22 +307,12 @@ def _row_from_aggregate( ) -> AtlasCell: """Build one AtlasCell row from drive-load aggregates.""" output_tps_per_gpu = ( - metrics.output_tps / identity.tensor_parallel - if identity.tensor_parallel - else metrics.output_tps - ) - total_tps_per_gpu = ( - metrics.total_tps / identity.tensor_parallel - if identity.tensor_parallel - else metrics.total_tps + metrics.output_tps / identity.tensor_parallel if identity.tensor_parallel else metrics.output_tps ) + total_tps_per_gpu = metrics.total_tps / identity.tensor_parallel if identity.tensor_parallel else metrics.total_tps # Mean ISL/OSL (shape) from the per-request token sums over OK requests. - mean_input_tokens = ( - metrics.total_input_tokens / metrics.n_ok if metrics.n_ok else None - ) - mean_output_tokens = ( - metrics.total_output_tokens / metrics.n_ok if metrics.n_ok else None - ) + mean_input_tokens = metrics.total_input_tokens / metrics.n_ok if metrics.n_ok else None + mean_output_tokens = metrics.total_output_tokens / metrics.n_ok if metrics.n_ok else None notes_parts = [ f"imported from {jsonl_path.parent.parent.parent.name}", @@ -427,9 +408,7 @@ def import_drive_load_bundle( if not bundle.is_dir(): raise ValueError(f"import_drive_load: bundle does not exist: {bundle}") - jsonl_files = _enumerate_jsonl_files( - bundle, concurrency_override=concurrency_override - ) + jsonl_files = _enumerate_jsonl_files(bundle, concurrency_override=concurrency_override) if not jsonl_files: raise ValueError( f"import_drive_load: no load.jsonl files found in {bundle} " @@ -437,12 +416,10 @@ def import_drive_load_bundle( ) bundle_meta = _load_bundle_metadata(bundle) - identity = _identity_from_meta_and_overrides( - bundle, bundle_meta, overrides or {} - ) + identity = _identity_from_meta_and_overrides(bundle, bundle_meta, overrides or {}) if captured_at is None: - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") rows: list[AtlasCell] = [] skipped: list[Path] = [] @@ -490,9 +467,7 @@ def import_drive_load_bundle( cell_dir.mkdir(parents=True, exist_ok=True) normalized_path = cell_dir / "normalized.json" - normalized_path.write_text( - json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True) - ) + normalized_path.write_text(json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True)) (cell_dir / "status.txt").write_text(status + "\n") (cell_dir / "backend.txt").write_text(BACKEND_VLLM_SWEEP + "\n") kernels_line = ( @@ -508,8 +483,7 @@ def import_drive_load_bundle( f"- concurrencies: {concurrencies}\n" f"- row_count: {len(rows)}\n" f"- status: {status}\n" - f"- skipped_files: {len(skipped)}\n" - + kernels_line + f"- skipped_files: {len(skipped)}\n" + kernels_line ) return DriveLoadImportResult( @@ -530,6 +504,7 @@ def import_drive_load_bundle( # Auto-detect dispatcher # ---------------------------------------------------------------------------- + def detect_bundle_pattern(bundle: Path) -> str: """Detect which importer to use for a given bundle directory. @@ -550,10 +525,7 @@ def detect_bundle_pattern(bundle: Path) -> str: raw_dir = bundle / "raw" if raw_dir.is_dir(): for entry in raw_dir.iterdir(): - if entry.is_file() and ( - entry.name.startswith("sweep-c") - or entry.name.startswith("sweep-K") - ): + if entry.is_file() and (entry.name.startswith("sweep-c") or entry.name.startswith("sweep-K")): return "inference_perf_bench" if (raw_dir / "load.jsonl").is_file(): return "inference_drive_load" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_perf_bench.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_perf_bench.py index b119919..503a692 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_perf_bench.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/inference_perf_bench.py @@ -45,25 +45,22 @@ import re import sys from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any from tools.perf_tune_report.helpers import resolve_cell_dir - +from tools.perf_tune_report.importers.zymtrace_kernels import ( + KernelImportResult, + import_zymtrace_kernels, +) from tools.perf_tune_report.schema import ( BACKEND_SGLANG_SWEEP, BACKEND_VLLM_SWEEP, - STATUS_FAILED, STATUS_FULL, STATUS_PARTIAL, AtlasCell, ) -from tools.perf_tune_report.importers.zymtrace_kernels import ( - KernelImportResult, - import_zymtrace_kernels, -) - # Regexes are anchored on the leading label so we don't pick up # percentile blocks or summaries from other parts of vllm's output. @@ -71,20 +68,12 @@ "n_reqs": re.compile(r"^Successful requests:\s+(\d+)\s*$", re.MULTILINE), "duration_s": re.compile(r"^Benchmark duration \(s\):\s+([\d.]+)\s*$", re.MULTILINE), "req_per_s": re.compile(r"^Request throughput \(req/s\):\s+([\d.]+)\s*$", re.MULTILINE), - "output_tps": re.compile( - r"^Output token throughput \(tok/s\):\s+([\d.]+)\s*$", re.MULTILINE - ), - "total_tps": re.compile( - r"^Total token throughput \(tok/s\):\s+([\d.]+)\s*$", re.MULTILINE - ), + "output_tps": re.compile(r"^Output token throughput \(tok/s\):\s+([\d.]+)\s*$", re.MULTILINE), + "total_tps": re.compile(r"^Total token throughput \(tok/s\):\s+([\d.]+)\s*$", re.MULTILINE), # Total input / generated tokens -> mean ISL/OSL per request (the shape # dimension pricing/capacity analysis needs). Derived against n_reqs below. - "total_input_tokens": re.compile( - r"^Total input tokens:\s+(\d+)\s*$", re.MULTILINE - ), - "total_generated_tokens": re.compile( - r"^Total generated tokens:\s+(\d+)\s*$", re.MULTILINE - ), + "total_input_tokens": re.compile(r"^Total input tokens:\s+(\d+)\s*$", re.MULTILINE), + "total_generated_tokens": re.compile(r"^Total generated tokens:\s+(\d+)\s*$", re.MULTILINE), "ttft_med_ms": re.compile(r"^Median TTFT \(ms\):\s+([\d.]+)\s*$", re.MULTILINE), "tpot_med_ms": re.compile(r"^Median TPOT \(ms\):\s+([\d.]+)\s*$", re.MULTILINE), } @@ -115,9 +104,7 @@ def _enumerate_sweep_files(bundle: Path) -> list[_SweepFileInfo]: continue m = _SWEEP_K.match(fp.name) if m: - files.append( - _SweepFileInfo(path=fp, concurrency=int(m.group(2)), k=int(m.group(1))) - ) + files.append(_SweepFileInfo(path=fp, concurrency=int(m.group(2)), k=int(m.group(1)))) continue m = _SWEEP_SIMPLE.match(fp.name) if m: @@ -267,6 +254,7 @@ def _identity_from_meta_and_overrides( > sensible defaults. Raises ``ValueError`` if a required field is still missing after the merge. """ + def pick(*keys: str, default: Any = None) -> Any: """Look up the first non-None match across overrides then bundle_meta. @@ -406,12 +394,8 @@ def _row_from_metrics( total_tps_total = metrics.get("total_tps") output_tps_per_user = (1000.0 / tpot_ms) if tpot_ms else None - output_tps_per_gpu = ( - (output_tps_total / identity.tensor_parallel) if output_tps_total else None - ) - total_tps_per_gpu = ( - (total_tps_total / identity.tensor_parallel) if total_tps_total else None - ) + output_tps_per_gpu = (output_tps_total / identity.tensor_parallel) if output_tps_total else None + total_tps_per_gpu = (total_tps_total / identity.tensor_parallel) if total_tps_total else None ttft = metrics.get("ttft_med_ms") # Mean ISL/OSL (shape): total input/generated tokens / successful requests. @@ -421,14 +405,17 @@ def _row_from_metrics( mean_input_tokens = (total_in / n_reqs) if (total_in and n_reqs) else None mean_output_tokens = (total_gen / n_reqs) if (total_gen and n_reqs) else None - # Typed num_speculative_tokens for THIS row (captured before the variant_extra loop below - # rebinds the local `k`): the row's K when sweeping K>1, else the bundle identity, else - # 1-if-MTP. This is what makes variant_key distinguish K=2 vs K=3. + # Typed num_speculative_tokens for THIS row: the row's K when sweeping K>1, + # else the bundle identity, else 1-if-MTP. This is what makes variant_key + # distinguish K=2 vs K=3. num_speculative_tokens = ( - k if (k and k > 1) - else (identity.speculative_num_tokens - if identity.speculative_num_tokens is not None - else (1 if identity.mtp else None)) + k + if (k and k > 1) + else ( + identity.speculative_num_tokens + if identity.speculative_num_tokens is not None + else (1 if identity.mtp else None) + ) ) notes_parts = [f"imported from {sweep_path.parent.parent.name}"] @@ -459,9 +446,9 @@ def _row_from_metrics( if identity.patch_files: extra["patch_files"] = identity.patch_files if identity.variant_extra: - for k, v in identity.variant_extra.items(): - if k not in extra: - extra[k] = v + for extra_key, extra_value in identity.variant_extra.items(): + if extra_key not in extra: + extra[extra_key] = extra_value # cell_id-with-K-suffix: keep different K runs distinguishable in the atlas. cell_id = identity.cell_id if k == 1 else f"{identity.cell_id}-K{k}" @@ -526,9 +513,7 @@ def _row_from_metrics( ) -def _check_plot_ready( - rows: list[AtlasCell], require_plot_ready: bool, *, context: str = "" -) -> None: +def _check_plot_ready(rows: list[AtlasCell], require_plot_ready: bool, *, context: str = "") -> None: """Fail EARLY (at import) when ``require_plot_ready`` and any STATUS_FULL row lacks the ttft + request-throughput a throughput scatter point needs. @@ -544,8 +529,7 @@ def _check_plot_ready( bad = [ f"{r.cell_id} (c={r.concurrency})" for r in rows - if r.status == STATUS_FULL - and (r.ttft_avg_ms is None or r.request_throughput_avg is None) + if r.status == STATUS_FULL and (r.ttft_avg_ms is None or r.request_throughput_avg is None) ] if bad: raise ValueError( @@ -555,8 +539,7 @@ def _check_plot_ready( f"output): {', '.join(bad)}. A strict throughput campaign cannot be " "built from incomplete capture. How to fix: re-capture the full, " "unfiltered `vllm bench serve` stdout and stderr. Do not grep-drop " - "those lines. Then re-import the bundle." - + (f" [{context}]" if context else "") + "those lines. Then re-import the bundle." + (f" [{context}]" if context else "") ) @@ -596,17 +579,13 @@ def import_perf_bench_bundle( sweep_files = _enumerate_sweep_files(bundle) if not sweep_files: - raise ValueError( - f"import_perf_bench: no sweep-c*.txt files found in {bundle}/raw/" - ) + raise ValueError(f"import_perf_bench: no sweep-c*.txt files found in {bundle}/raw/") bundle_meta = _load_bundle_metadata(bundle) - identity = _identity_from_meta_and_overrides( - bundle, bundle_meta, overrides or {} - ) + identity = _identity_from_meta_and_overrides(bundle, bundle_meta, overrides or {}) if captured_at is None: - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") rows: list[AtlasCell] = [] skipped: list[Path] = [] @@ -671,9 +650,7 @@ def import_perf_bench_bundle( cell_dir.mkdir(parents=True, exist_ok=True) normalized_path = cell_dir / "normalized.json" - normalized_path.write_text( - json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True) - ) + normalized_path.write_text(json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True)) (cell_dir / "status.txt").write_text(status + "\n") (cell_dir / "backend.txt").write_text(BACKEND_VLLM_SWEEP + "\n") kernels_line = ( @@ -689,8 +666,7 @@ def import_perf_bench_bundle( f"- k_values: {k_values}\n" f"- row_count: {len(rows)}\n" f"- status: {status}\n" - f"- skipped_files: {len(skipped)}\n" - + kernels_line + f"- skipped_files: {len(skipped)}\n" + kernels_line ) return ImportResult( diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/lws_summary.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/lws_summary.py index 927afd8..f8ced1b 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/lws_summary.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/lws_summary.py @@ -53,7 +53,7 @@ import dataclasses import json import re -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -63,7 +63,6 @@ AtlasCell, ) - _KNOB_MBT_RE = re.compile(r"mbt=(\d+)") @@ -168,27 +167,19 @@ def import_lws_summary_bundle( overrides = overrides or {} if not bundle.is_dir(): - raise ValueError( - f"import_lws_summary: bundle directory does not exist: {bundle}" - ) + raise ValueError(f"import_lws_summary: bundle directory does not exist: {bundle}") summary_path = bundle / "summary.json" if not summary_path.is_file(): - raise ValueError( - f"import_lws_summary: summary.json not found at {summary_path}" - ) + raise ValueError(f"import_lws_summary: summary.json not found at {summary_path}") try: summary = json.loads(summary_path.read_text()) except json.JSONDecodeError as e: - raise ValueError( - f"import_lws_summary: summary.json malformed at {summary_path}: {e}" - ) from e + raise ValueError(f"import_lws_summary: summary.json malformed at {summary_path}: {e}") from e variants = summary.get("variants") if not isinstance(variants, list) or not variants: - raise ValueError( - f"import_lws_summary: summary.json at {summary_path} has no variants[]" - ) + raise ValueError(f"import_lws_summary: summary.json at {summary_path} has no variants[]") # Resolve overrides + defaults. model = overrides.get("model", "zai-org/GLM-5.1") @@ -233,6 +224,8 @@ def import_lws_summary_bundle( if not isinstance(m, dict): continue c_raw = m.get("concurrency", m.get("c")) + if c_raw is None: + continue try: c = int(c_raw) except (TypeError, ValueError): diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/model_eval.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/model_eval.py index 1eceb93..b00ea94 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/model_eval.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/model_eval.py @@ -12,6 +12,7 @@ Dependency-light (stdlib + schema only). """ + from __future__ import annotations import json diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/ncu_kernels.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/ncu_kernels.py index bfbb1b3..5deb495 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/ncu_kernels.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/ncu_kernels.py @@ -87,9 +87,16 @@ # name), so a unit-unaware sum is off by 1e3/1e6/1e9. This maps the unit to a # byte multiplier (ncu uses decimal SI prefixes; binary Ki/Mi/Gi handled too). _BYTE_UNIT_SCALE: dict[str, float] = { - "byte": 1.0, "bytes": 1.0, - "kbyte": 1e3, "mbyte": 1e6, "gbyte": 1e9, "tbyte": 1e12, - "kib": 1024.0, "mib": 1024.0**2, "gib": 1024.0**3, "tib": 1024.0**4, + "byte": 1.0, + "bytes": 1.0, + "kbyte": 1e3, + "mbyte": 1e6, + "gbyte": 1e9, + "tbyte": 1e12, + "kib": 1024.0, + "mib": 1024.0**2, + "gib": 1024.0**3, + "tib": 1024.0**4, } # Tensor-core "ops" -> FLOPs. ncu's sm__ops_path_tensor_*.sum counts each math @@ -115,10 +122,17 @@ def _byte_scale(unit: str | None) -> float: # a correct FLOPS-rate denominator. The SoL-section Duration is normalised # separately by _duration_ns; this covers the no-SoL path. _TIME_UNIT_SCALE: dict[str, float] = { - "ns": 1.0, "nsecond": 1.0, "nanosecond": 1.0, - "us": 1e3, "usecond": 1e3, "microsecond": 1e3, - "ms": 1e6, "msecond": 1e6, "millisecond": 1e6, - "s": 1e9, "second": 1e9, + "ns": 1.0, + "nsecond": 1.0, + "nanosecond": 1.0, + "us": 1e3, + "usecond": 1e3, + "microsecond": 1e3, + "ms": 1e6, + "msecond": 1e6, + "millisecond": 1e6, + "s": 1e9, + "second": 1e9, } @@ -169,9 +183,7 @@ class NcuImportResult: def to_dict(self) -> dict[str, Any]: return { "bundle": str(self.bundle), - "ncu_kernels_json_path": ( - str(self.ncu_kernels_json_path) if self.ncu_kernels_json_path else None - ), + "ncu_kernels_json_path": (str(self.ncu_kernels_json_path) if self.ncu_kernels_json_path else None), "skipped_reason": self.skipped_reason, "kernel_count": self.kernel_count, } @@ -207,7 +219,7 @@ def _pivot_long_rows(rows: list[dict[str, str]]) -> list[dict[str, str]]: lookups match. Rows with an empty Metric Name (ncu's per-section rule rows, e.g. ``SOLBottleneck``) are skipped. """ - wide: "OrderedDict[tuple[str, str], dict[str, str]]" = OrderedDict() + wide: OrderedDict[tuple[str, str], dict[str, str]] = OrderedDict() for r in rows: name = (r.get("Kernel Name") or "").strip() metric = (r.get("Metric Name") or "").strip() @@ -266,9 +278,7 @@ def _read_ncu_csv(path: Path) -> tuple[list[dict[str, str]], dict[str, str]]: header_idx = i break if header_idx is None: - raise NcuCsvMalformed( - path, reason="no header row with 'Kernel Name' column" - ) + raise NcuCsvMalformed(path, reason="no header row with 'Kernel Name' column") reader = csv.DictReader(lines[header_idx:]) fieldnames = reader.fieldnames or [] @@ -279,10 +289,9 @@ def _read_ncu_csv(path: Path) -> tuple[list[dict[str, str]], dict[str, str]]: if "Metric Name" in fieldnames and "Metric Value" in fieldnames: pivoted = _pivot_long_rows(rows) if not pivoted: - raise NcuCsvMalformed( - path, reason="long-format CSV has no (kernel, metric) rows" - ) + raise NcuCsvMalformed(path, reason="long-format CSV has no (kernel, metric) rows") return pivoted, {} + # ncu 2026.1.1 emits a units-row between the header and the first data # row (e.g. dram bytes column shows "Mbyte" instead of a numeric value, # Kernel Name column is empty). Capture that units row (keyed by column) @@ -401,32 +410,22 @@ def _aggregate_per_kernel( for r in sol_rows: name = r.get("Kernel Name") or r.get("kernel_name") or r.get("ID") if not name: - raise NcuCsvMalformed( - Path(""), reason=f"row missing 'Kernel Name': {list(r.keys())[:5]}" - ) + raise NcuCsvMalformed(Path(""), reason=f"row missing 'Kernel Name': {list(r.keys())[:5]}") sol_by_kernel.setdefault(name, []).append(r) raw_by_kernel: dict[str, list[dict[str, str]]] = {} for r in raw_rows: name = r.get("Kernel Name") or r.get("kernel_name") or r.get("ID") if not name: - raise NcuCsvMalformed( - Path(""), reason=f"row missing 'Kernel Name': {list(r.keys())[:5]}" - ) + raise NcuCsvMalformed(Path(""), reason=f"row missing 'Kernel Name': {list(r.keys())[:5]}") raw_by_kernel.setdefault(name, []).append(r) out: list[dict[str, Any]] = [] for name, sol_group in sol_by_kernel.items(): # Mean across launches for percentages. - dram_pct = _mean_optional([ - _parse_float(_find_column(r, "DRAM Throughput", "%")) for r in sol_group - ]) - sm_pct = _mean_optional([ - _parse_float(_find_column(r, "Compute (SM) Throughput", "%")) for r in sol_group - ]) - occupancy = _mean_optional([ - _parse_float(_find_column(r, "Achieved Occupancy")) for r in sol_group - ]) + dram_pct = _mean_optional([_parse_float(_find_column(r, "DRAM Throughput", "%")) for r in sol_group]) + sm_pct = _mean_optional([_parse_float(_find_column(r, "Compute (SM) Throughput", "%")) for r in sol_group]) + occupancy = _mean_optional([_parse_float(_find_column(r, "Achieved Occupancy")) for r in sol_group]) # SoL Duration (normalised to ns); preferred over the raw-page time # below because it carries an explicit unit. None when the SoL # section / Duration metric is absent. @@ -444,12 +443,10 @@ def _aggregate_per_kernel( # "Achieved Occupancy"; the raw page carries the equivalent # sm__warps_active.avg.pct_of_peak_sustained_active. if occupancy is None: - occupancy = _mean_optional([ - _parse_float( - _find_column(r, "sm__warps_active.avg.pct_of_peak_sustained_active") - ) - for r in raw_group - ]) + occupancy = _mean_optional( + [_parse_float(_find_column(r, "sm__warps_active.avg.pct_of_peak_sustained_active")) for r in raw_group] + ) + # DRAM bytes, unit-scaled to bytes (the units row gave e.g. "Mbyte"). # ncu expands a requested counter into .avg/.max/.min/.sum columns; the # per-kernel TOTAL is the .sum column. Match that specifically -- a naive @@ -509,9 +506,7 @@ def _dram_bytes_scaled(r: dict[str, str]) -> float | None: if had_tensor: tensor_per_launch.append(tensor) sm_flops_total = sum(scalar_per_launch) if scalar_per_launch else None - tensor_flops_total = ( - sum(tensor_per_launch) * _TENSOR_OP_TO_FLOP if tensor_per_launch else None - ) + tensor_flops_total = sum(tensor_per_launch) * _TENSOR_OP_TO_FLOP if tensor_per_launch else None # Wall-clock time. Two distinct quantities: # - time_ns_sum: active time SUMMED across launches -- the correct @@ -538,9 +533,7 @@ def _raw_time_ns(r: dict[str, str]) -> float | None: # version-dependent unit (often usecond) that the raw page does NOT # unit-convert, so using it directly would inflate the rate ~1000x. n_launch = len(raw_group) or len(sol_group) or 1 - rate_time_ns = ( - sol_time_ns * n_launch if sol_time_ns is not None else time_ns_sum - ) + rate_time_ns = sol_time_ns * n_launch if sol_time_ns is not None else time_ns_sum # Total FLOPs = scalar (CUDA-core) + tensor (MMA). Either may be None; # combine the present ones. For a tensor-core kernel this is dominated @@ -562,21 +555,23 @@ def _raw_time_ns(r: dict[str, str]) -> float | None: # kept under "name_full" for traceability. short_name = re.split(r"[<(]", name, maxsplit=1)[0] - out.append({ - "name": short_name, - "name_full": name, - "category": _categorize(name), - "kernel_time_ns": time_ns, - "dram_bytes_total": dram_bytes, - "sm_flops_total": sm_flops_total, - "tensor_flops_total": tensor_flops_total, - "arithmetic_intensity_flops_per_byte": ai, - "achieved_dram_pct_peak": dram_pct, - "achieved_sm_pct_peak": sm_pct, - "achieved_occupancy_pct": occupancy, - "block_limit_factor": block_limit, - "achieved_tflops": achieved_tflops, - }) + out.append( + { + "name": short_name, + "name_full": name, + "category": _categorize(name), + "kernel_time_ns": time_ns, + "dram_bytes_total": dram_bytes, + "sm_flops_total": sm_flops_total, + "tensor_flops_total": tensor_flops_total, + "arithmetic_intensity_flops_per_byte": ai, + "achieved_dram_pct_peak": dram_pct, + "achieved_sm_pct_peak": sm_pct, + "achieved_occupancy_pct": occupancy, + "block_limit_factor": block_limit, + "achieved_tflops": achieved_tflops, + } + ) return out diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/roofline_sweep.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/roofline_sweep.py index 8a869ae..448c949 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/roofline_sweep.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/roofline_sweep.py @@ -29,13 +29,12 @@ import json from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any -from tools.perf_tune_report.helpers import resolve_cell_dir, safe_path_segment - from tools.perf_tune_report import roofline_math +from tools.perf_tune_report.helpers import resolve_cell_dir, safe_path_segment from tools.perf_tune_report.schema import ( BACKEND_VLLM_SWEEP, STATUS_FULL, @@ -52,7 +51,7 @@ def _resolve_shape( bundle: Path, identity: dict[str, Any], overrides: dict[str, Any] -) -> "roofline_math.ModelShape | None": +) -> roofline_math.ModelShape | None: """Resolve the analytical ModelShape used to embed FLOP/byte + byte-grounded fields. Priority: explicit --model-config path > a model_config.json captured in the bundle > the built-in registry (by served model name).""" @@ -123,7 +122,7 @@ def _dcgm_util(cell: dict[str, Any]) -> dict[str, float | int | None]: def _point_analytics( cell: dict[str, Any], phase: str, - shape: "roofline_math.ModelShape | None", + shape: roofline_math.ModelShape | None, quant: str, tp: int, kv_dtype: str, @@ -136,28 +135,23 @@ def _point_analytics( delivered HBM bytes/s / GPU. The %-of-peak is computed downstream where the sol-ceilings.yaml peaks are loaded (renderer + lake_writer).""" if shape is None: - return {"arithmetic_intensity": None, "achieved_tflops_per_gpu": None, - "hbm_delivered_Bps_per_gpu": None} + return {"arithmetic_intensity": None, "achieved_tflops_per_gpu": None, "hbm_delivered_Bps_per_gpu": None} b = cell.get("bench", {}) or {} if phase == "decode": c = cell.get("c") or 1 rate = b.get("output_throughput") if not rate: - return {"arithmetic_intensity": None, "achieved_tflops_per_gpu": None, - "hbm_delivered_Bps_per_gpu": None} + return {"arithmetic_intensity": None, "achieved_tflops_per_gpu": None, "hbm_delivered_Bps_per_gpu": None} ctx = int((cell.get("isl") or 256) + (cell.get("osl") or 512) // 2) ai = shape.decode_arithmetic_intensity(c, ctx, quant, kv_dtype) - union = (min(shape.n_routed_experts, shape.n_experts_per_tok * c) - if shape.is_moe else 0) - bytes_per_tok = shape.active_weight_bytes(union, quant) / c \ - + shape.kv_bytes_per_token(ctx, kv_dtype) + union = min(shape.n_routed_experts, shape.n_experts_per_tok * c) if shape.is_moe else 0 + bytes_per_tok = shape.active_weight_bytes(union, quant) / c + shape.kv_bytes_per_token(ctx, kv_dtype) else: # prefill isl = cell.get("isl") inp, dur = b.get("total_input_tokens"), b.get("duration") rate = (inp / dur) if (inp and dur) else None if not rate or not isl: - return {"arithmetic_intensity": None, "achieved_tflops_per_gpu": None, - "hbm_delivered_Bps_per_gpu": None} + return {"arithmetic_intensity": None, "achieved_tflops_per_gpu": None, "hbm_delivered_Bps_per_gpu": None} ai = shape.prefill_arithmetic_intensity(isl, quant) experts = shape.n_routed_experts if shape.is_moe else 0 bytes_per_tok = shape.active_weight_bytes(experts, quant) / max(int(isl), 1) @@ -175,7 +169,7 @@ def _atlas_row( identity: dict[str, Any], captured_at: str, raw_path: str, - shape: "roofline_math.ModelShape | None" = None, + shape: roofline_math.ModelShape | None = None, ) -> AtlasCell | None: b = cell.get("bench", {}) or {} if not b: @@ -185,8 +179,7 @@ def _atlas_row( out_tps = b.get("output_throughput") total_tps = b.get("total_token_throughput") util = _dcgm_util(cell) - analytics = _point_analytics(cell, phase, shape, identity["quant"], tp, - identity.get("kv_dtype", "fp8")) + analytics = _point_analytics(cell, phase, shape, identity["quant"], tp, identity.get("kv_dtype", "fp8")) _c = cell.get("c") or 0 _np = cell.get("num_prompts") or 0 # decode steady-window guard: num_prompts must be >= 2*c so the bench's @@ -249,7 +242,7 @@ def _roofline_artifact( decode: list[dict[str, Any]], prefill: list[dict[str, Any]], identity: dict[str, Any], - shape: "roofline_math.ModelShape | None" = None, + shape: roofline_math.ModelShape | None = None, ) -> dict[str, Any]: """The phase-tagged operating points the renderer page consumes. Carries the embedded ``analytical_shape`` (so the renderer + lake are self-contained, @@ -281,6 +274,7 @@ def point(cell: dict[str, Any], phase: str) -> dict[str, Any]: "achieved_tflops_per_gpu": a["achieved_tflops_per_gpu"], "hbm_delivered_Bps_per_gpu": a["hbm_delivered_Bps_per_gpu"], } + art: dict[str, Any] = { "schema": "roofline_sweep_points_v1", "hardware": identity["hardware"], @@ -327,16 +321,22 @@ def pick(*keys, default=None): if cudagraph_mode == "unknown" and pick("enforce_eager") is True: cudagraph_mode = "eager" gmu = pick("gpu_memory_utilization") + tensor_parallel = pick("tensor_parallel", default=4) + if tensor_parallel is None: + tensor_parallel = 4 + max_num_batched_tokens = pick("max_num_batched_tokens", default=12288) + if max_num_batched_tokens is None: + max_num_batched_tokens = 12288 return { "cell_id": overrides.get("cell_id") or bundle.name, "model": model, "hardware": hardware, "quant": quant, "kv_dtype": pick("kv_dtype", "kv_cache_dtype", default="fp8"), - "tensor_parallel": int(pick("tensor_parallel", default=4)), + "tensor_parallel": int(tensor_parallel), "parallel_strategy": pick("parallel_strategy", default="TP"), "mtp": bool(pick("mtp", default=False)), - "max_num_batched_tokens": int(pick("max_num_batched_tokens", default=12288)), + "max_num_batched_tokens": int(max_num_batched_tokens), "cache_mode": pick("cache_mode", default="unknown"), "dataset": pick("dataset", default="random"), "cudagraph_mode": cudagraph_mode, @@ -361,8 +361,7 @@ def _steady_window_warnings(decode: list[dict[str, Any]]) -> list[dict[str, Any] c = cell.get("c") or 0 npr = cell.get("num_prompts") or 0 if c >= 16 and npr and npr < 2 * c: - warns.append({"c": c, "num_prompts": npr, "min_required": 2 * c, - "tag": cell.get("tag")}) + warns.append({"c": c, "num_prompts": npr, "min_required": 2 * c, "tag": cell.get("tag")}) return warns @@ -382,8 +381,8 @@ def import_roofline_sweep_bundle( steady_warns = _steady_window_warnings(decode) if steady_warns: import sys as _sys - cells = ", ".join(f"c={w['c']}(num={w['num_prompts']}<{w['min_required']})" - for w in steady_warns) + + cells = ", ".join(f"c={w['c']}(num={w['num_prompts']}<{w['min_required']})" for w in steady_warns) print( f"WARN: steady-window undercount in {len(steady_warns)} decode cell(s) " f"[{cells}] -- num_prompts < 2*c, so output_throughput is ramp/drain-" @@ -393,14 +392,12 @@ def import_roofline_sweep_bundle( file=_sys.stderr, ) if not decode and not prefill: - raise ValueError( - f"import_roofline_sweep: no points in {bundle}/{DECODE_FILE} or {PREFILL_FILE}" - ) + raise ValueError(f"import_roofline_sweep: no points in {bundle}/{DECODE_FILE} or {PREFILL_FILE}") overrides = overrides or {} identity = _resolve_identity(bundle, overrides) shape = _resolve_shape(bundle, identity, overrides) if captured_at is None: - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") base = safe_path_segment(identity["cell_id"], label="cell-id") cell_dirs: list[str] = [] @@ -409,11 +406,14 @@ def import_roofline_sweep_bundle( if not cells: continue rows = [ - r for r in ( - _atlas_row(c, phase, f"{base}-{phase}", identity, captured_at, - str(bundle / f"{phase}_sweep.jsonl"), shape) + r + for r in ( + _atlas_row( + c, phase, f"{base}-{phase}", identity, captured_at, str(bundle / f"{phase}_sweep.jsonl"), shape + ) for c in cells - ) if r is not None + ) + if r is not None ] if not rows: continue @@ -422,9 +422,7 @@ def import_roofline_sweep_bundle( if dry_run: continue cell_dir.mkdir(parents=True, exist_ok=True) - (cell_dir / "normalized.json").write_text( - json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True) - ) + (cell_dir / "normalized.json").write_text(json.dumps([r.to_dict() for r in rows], indent=2, sort_keys=True)) (cell_dir / "status.txt").write_text(STATUS_FULL + "\n") (cell_dir / "backend.txt").write_text(BACKEND_VLLM_SWEEP + "\n") # the renderer-page input artifact (written on the decode cell, carrying both phases) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/variant_ab.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/variant_ab.py index da5cb34..2d92c1f 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/variant_ab.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/variant_ab.py @@ -30,19 +30,11 @@ import re import statistics from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any from tools.perf_tune_report.helpers import resolve_cell_dir - -from tools.perf_tune_report.schema import ( - BACKEND_SGLANG_SWEEP, - BACKEND_VLLM_SWEEP, - STATUS_FULL, - STATUS_PARTIAL, - AtlasCell, -) from tools.perf_tune_report.importers.inference_perf_bench import ( _CellIdentity, _check_plot_ready, @@ -50,6 +42,13 @@ _row_from_metrics, ) from tools.perf_tune_report.importers.zymtrace_kernels import import_zymtrace_kernels +from tools.perf_tune_report.schema import ( + BACKEND_SGLANG_SWEEP, + BACKEND_VLLM_SWEEP, + STATUS_FULL, + STATUS_PARTIAL, + AtlasCell, +) # A raw per-trial file: ``c-t.txt``. _TRIAL_RE = re.compile(r"^c(\d+)-t(\d+)\.txt$") @@ -147,9 +146,7 @@ def _avg_metrics(trial_files: list[Path]) -> dict[str, float | int] | None: return avg -def _arm_identity( - arm: Path, overrides: dict[str, Any] -) -> _CellIdentity: +def _arm_identity(arm: Path, overrides: dict[str, Any]) -> _CellIdentity: res: dict[str, Any] = {} rj = arm / "result.json" if rj.is_file(): @@ -157,6 +154,7 @@ def _arm_identity( res = json.loads(rj.read_text()) except json.JSONDecodeError: res = {} + def pick(*keys: str, default: Any = None) -> Any: for key in keys: if key in overrides and overrides[key] is not None: @@ -168,27 +166,24 @@ def pick(*keys: str, default: Any = None) -> Any: model = pick("model") if not model: - raise ValueError( - "import_perf_bench (variant_ab): --model is required " - "(arm result.json carries no model)" - ) + raise ValueError("import_perf_bench (variant_ab): --model is required (arm result.json carries no model)") hw = pick("hardware", default="B200") hardware = hw.split(" ")[0] if isinstance(hw, str) else hw tp = int(pick("tensor_parallel", "tp", default=8)) warm = res.get("warm") cache_mode = ( - "warm" if warm is True - else "cold" if warm is False - else (pick("cache_mode", default="unknown") or "unknown") + "warm" if warm is True else "cold" if warm is False else (pick("cache_mode", default="unknown") or "unknown") ) if cache_mode not in ("warm", "cold", "unknown"): cache_mode = "unknown" mtp = ( - bool(overrides["mtp"]) if overrides.get("mtp") is not None - else bool(res["mtp"]) if res.get("mtp") is not None + bool(overrides["mtp"]) + if overrides.get("mtp") is not None + else bool(res["mtp"]) + if res.get("mtp") is not None else "mtp" in arm.name.lower() ) - # Engine (cross-engine A/B): result.json "engine" from the producer. + # Engine (cross-engine A/B): result.json "engine" from the producer. # wins; else CLI override; else infer from the arm name (the "-s-" / "sgl" naming # convention); else default vllm. engine = ( @@ -202,11 +197,7 @@ def pick(*keys: str, default: Any = None) -> Any: cudagraph_mode = "eager" gmu = pick("gpu_memory_utilization", "gpu_memory_util") pchr = pick("prefix_cache_hit_rate", "gpu_prefix_cache_hit_rate") - variant_extra = { - k: res[k] - for k in _VARIANT_EXTRA_KEYS - if k in res and res[k] not in (None, "", "unknown") - } + variant_extra = {k: res[k] for k in _VARIANT_EXTRA_KEYS if k in res and res[k] not in (None, "", "unknown")} extra_block = res.get("extra") if isinstance(extra_block, dict): for k, v in extra_block.items(): @@ -259,12 +250,10 @@ def import_variant_ab_bundle( raise ValueError(f"import_perf_bench (variant_ab): bundle does not exist: {bundle}") arms = _arm_dirs(bundle) if not arms: - raise ValueError( - f"import_perf_bench (variant_ab): no /c-t.txt dirs in {bundle}" - ) + raise ValueError(f"import_perf_bench (variant_ab): no /c-t.txt dirs in {bundle}") overrides = overrides or {} if captured_at is None: - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") cells: list[str] = [] total_rows = 0 @@ -332,14 +321,11 @@ def import_variant_ab_bundle( f"- captured_at: {captured_at}\n" f"- concurrencies: {sorted(by_c)}\n" f"- trials/conc: {{c: len(files)}} averaged\n" - f"- row_count: {len(rows)}\n" - + kernels_line + f"- row_count: {len(rows)}\n" + kernels_line ) if not cells: - raise ValueError( - f"import_perf_bench (variant_ab): no arm produced rows in {bundle}" - ) + raise ValueError(f"import_perf_bench (variant_ab): no arm produced rows in {bundle}") return VariantAbImportResult( campaign_dir=campaign_dir, bundle_path=bundle, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/workloads.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/workloads.py index fe9f6b8..3df81d5 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/workloads.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/workloads.py @@ -11,12 +11,13 @@ The serve-config identity (model / hardware / TP / kv-cache / image / ...) is NOT in the bench output, so it is supplied by the caller (the same overrides ``import_perf_bench`` takes). """ + from __future__ import annotations import json import re from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -184,7 +185,7 @@ def import_workloads( continue by_tag.setdefault(m.group("tag"), []).append((int(m.group("c")), fp)) - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") tags_done: list[str] = [] skipped: list[str] = [] n_rows = 0 diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/zymtrace_kernels.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/zymtrace_kernels.py index e90d88d..d257555 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/zymtrace_kernels.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/importers/zymtrace_kernels.py @@ -63,11 +63,11 @@ from __future__ import annotations import json +import re as _re from dataclasses import dataclass from pathlib import Path from typing import Any - # Expected TSV filenames. Order matches the five capture queries. _TSV_FILES = [ "kernel-class.tsv", @@ -142,7 +142,7 @@ class KernelImportResult: bundle: Path kernels_json_path: Path | None # None when manifest doesn't declare zymtrace - skipped_reason: str | None # set when kernels_json_path is None + skipped_reason: str | None # set when kernels_json_path is None top_kernel_count: int category_count: int gpu_count: int @@ -197,18 +197,12 @@ def _read_tsv(path: Path) -> list[dict[str, str]]: path, reason=f"line {ln_no}: column count {len(cells)} != header {len(header)}", ) - rows.append(dict(zip(header, cells))) + rows.append(dict(zip(header, cells, strict=True))) if not rows: raise ZymtraceTSVMalformed(path, reason="no data rows after header") return rows -# CSV-friendly regex categorizer for top_kernels emission. Mirror of the -# capture-query multiIf bucketing so -# that JSON consumers can re-derive a kernel's category from its name -# without re-querying the per-category TSV. -import re as _re - _CATEGORY_RULES: list[tuple[_re.Pattern[str], str]] = [ (_re.compile(r"(?i)(multimem|allreduce|flashinfer.*allreduce|nccl)"), "NCCL"), (_re.compile(r"(?i)(routingIndices|finalizeKernel|moe)"), "MoE"), @@ -234,13 +228,9 @@ def _build_top_kernels(rows: list[dict[str, str]]) -> list[dict[str, Any]]: try: samples = int(r["samples"]) except (KeyError, ValueError) as e: - raise ZymtraceTSVMalformed( - Path(""), reason=f"bad samples cell in row {r}: {e}" - ) from e + raise ZymtraceTSVMalformed(Path(""), reason=f"bad samples cell in row {r}: {e}") from e if "kernel" not in r: - raise ZymtraceTSVMalformed( - Path(""), reason=f"missing kernel column in row {r}" - ) + raise ZymtraceTSVMalformed(Path(""), reason=f"missing kernel column in row {r}") # An empty value is a legitimate UNSYMBOLIZED frame (profiling data, not # malformed) -- keep it as rather than crashing the import. name = r.get("kernel") or "" @@ -254,14 +244,14 @@ def _build_per_gpu(rows: list[dict[str, str]]) -> list[dict[str, Any]]: try: samples = int(r["samples"]) except (KeyError, ValueError) as e: - raise ZymtraceTSVMalformed( - Path(""), reason=f"bad samples cell in row {r}: {e}" - ) from e - out.append({ - "gpu_name": r.get("gpu_name", ""), - "gpu_uuid": r.get("gpu_uuid", ""), - "samples": samples, - }) + raise ZymtraceTSVMalformed(Path(""), reason=f"bad samples cell in row {r}: {e}") from e + out.append( + { + "gpu_name": r.get("gpu_name", ""), + "gpu_uuid": r.get("gpu_uuid", ""), + "samples": samples, + } + ) return out @@ -270,15 +260,11 @@ def _build_per_category(rows: list[dict[str, str]]) -> dict[str, int]: for r in rows: cat = r.get("category", "") if not cat: - raise ZymtraceTSVMalformed( - Path(""), reason=f"missing category column in row {r}" - ) + raise ZymtraceTSVMalformed(Path(""), reason=f"missing category column in row {r}") try: samples = int(r["samples"]) except (KeyError, ValueError) as e: - raise ZymtraceTSVMalformed( - Path(""), reason=f"bad samples cell in row {r}: {e}" - ) from e + raise ZymtraceTSVMalformed(Path(""), reason=f"bad samples cell in row {r}: {e}") from e out[cat] = samples return out diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_profile/kernel_profile.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_profile/kernel_profile.py index d070830..491a1af 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_profile/kernel_profile.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_profile/kernel_profile.py @@ -77,12 +77,13 @@ import subprocess import time from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any - -_DEFAULT_SIDECAR_IMAGE = "ghcr.io/cfregly/nsys-sidecar:0.1.0@sha256:3146de96f6022a8cc36f86d1b8c0281cb940e51e2c3dc49c315646ad66ede43d" +_DEFAULT_SIDECAR_IMAGE = ( + "ghcr.io/cfregly/nsys-sidecar:0.1.0@sha256:3146de96f6022a8cc36f86d1b8c0281cb940e51e2c3dc49c315646ad66ede43d" +) _DEFAULT_DURATION_S = 120 _DEFAULT_VLLM_PID_PATTERN = "vllm serve" _DEFAULT_SAMPLE = "cpu" @@ -111,7 +112,7 @@ class KernelProfileStepFns: extract_artifacts: Any = None @classmethod - def production(cls) -> "KernelProfileStepFns": + def production(cls) -> KernelProfileStepFns: return cls( validate_pod=_step_validate_pod, attach_sidecar=_step_attach_sidecar, @@ -171,9 +172,7 @@ def _step_validate_pod(*, namespace: str, pod: str, target_container: str) -> di check=False, ) if result.returncode != 0: - raise RuntimeError( - f"validate_pod: pod {namespace}/{pod} not found: {result.stderr.strip()}" - ) + raise RuntimeError(f"validate_pod: pod {namespace}/{pod} not found: {result.stderr.strip()}") pod_obj = json.loads(result.stdout) containers = [c["name"] for c in pod_obj.get("spec", {}).get("containers", [])] if target_container not in containers: @@ -198,14 +197,19 @@ def _step_attach_sidecar( ) -> dict[str, Any]: """``kubectl debug --share-processes`` to attach the nsys sidecar.""" args = [ - "-n", namespace, - "debug", pod, - "--image", sidecar_image, - "--container", sidecar_container, + "-n", + namespace, + "debug", + pod, + "--image", + sidecar_image, + "--container", + sidecar_container, "--share-processes", f"--target={target_container}", "--", - "sleep", "3600", + "sleep", + "3600", ] _run_kubectl(args) return { @@ -243,8 +247,7 @@ def _step_wait_for_sidecar( } time.sleep(_SIDECAR_POLL_INTERVAL_S) raise RuntimeError( - f"wait_for_sidecar: ephemeral container {sidecar_container} did not " - f"become Running within {timeout_s}s" + f"wait_for_sidecar: ephemeral container {sidecar_container} did not become Running within {timeout_s}s" ) @@ -257,18 +260,21 @@ def _step_find_vllm_pid( ) -> dict[str, Any]: """``kubectl exec`` + pgrep to find the engine PID.""" args = [ - "-n", namespace, - "exec", pod, "-c", sidecar_container, - "--", "bash", "-c", + "-n", + namespace, + "exec", + pod, + "-c", + sidecar_container, + "--", + "bash", + "-c", f"pgrep -f {shlex.quote(vllm_pid_pattern)} | head -1", ] result = _run_kubectl(args) pid_str = result.stdout.strip() if not pid_str: - raise RuntimeError( - f"find_vllm_pid: no process matching {vllm_pid_pattern!r} in pod " - f"{namespace}/{pod}" - ) + raise RuntimeError(f"find_vllm_pid: no process matching {vllm_pid_pattern!r} in pod {namespace}/{pod}") return { "cmd": "kubectl " + " ".join(shlex.quote(a) for a in args), "vllm_pid": int(pid_str), @@ -304,9 +310,16 @@ def _step_run_nsys_profile( f"--output=/profiling/{shlex.quote(capture_name)}" ) args = [ - "-n", namespace, - "exec", pod, "-c", sidecar_container, - "--", "bash", "-c", nsys_cmd, + "-n", + namespace, + "exec", + pod, + "-c", + sidecar_container, + "--", + "bash", + "-c", + nsys_cmd, ] _run_kubectl(args) return { @@ -327,19 +340,20 @@ def _step_extract_artifacts( """``kubectl cp`` to extract .nsys-rep + summary CSVs into ``output_dir``.""" output_dir.mkdir(parents=True, exist_ok=True) in_pod_to_local = { - f"/profiling/{capture_name}.nsys-rep": - output_dir / f"{capture_name}.nsys-rep", - f"/profiling/{capture_name}_gpu_kern_sum.csv": - output_dir / f"{capture_name}_gpu_kern_sum.csv", - f"/profiling/{capture_name}_cuda_api_sum.csv": - output_dir / f"{capture_name}_cuda_api_sum.csv", + f"/profiling/{capture_name}.nsys-rep": output_dir / f"{capture_name}.nsys-rep", + f"/profiling/{capture_name}_gpu_kern_sum.csv": output_dir / f"{capture_name}_gpu_kern_sum.csv", + f"/profiling/{capture_name}_cuda_api_sum.csv": output_dir / f"{capture_name}_cuda_api_sum.csv", } cmds: list[str] = [] for in_pod_path, local_path in in_pod_to_local.items(): args = [ - "-n", namespace, - "cp", f"{pod}:{in_pod_path}", str(local_path), - "-c", sidecar_container, + "-n", + namespace, + "cp", + f"{pod}:{in_pod_path}", + str(local_path), + "-c", + sidecar_container, ] _run_kubectl(args) cmds.append("kubectl " + " ".join(shlex.quote(a) for a in args)) @@ -405,7 +419,7 @@ def capture_kernel_profile( PID not found, nsys exit error) """ if captured_at is None: - captured_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + captured_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") if sidecar_container is None: sidecar_container = f"nsys-debug-{int(time.time())}" if capture_name is None: @@ -420,9 +434,7 @@ def capture_kernel_profile( # Step 1: validate pod (always runs, even in dry-run, so we fail fast on # bad inputs). - validate_out = step_fns.validate_pod( - namespace=namespace, pod=pod, target_container=target_container - ) + validate_out = step_fns.validate_pod(namespace=namespace, pod=pod, target_container=target_container) step_commands.append({"step": "validate_pod", **validate_out}) if dry_run: @@ -559,9 +571,7 @@ def capture_kernel_profile( "method": "kubectl-debug-share-processes", } kernel_profile_json_path = output_dir / "kernel_profile.json" - kernel_profile_json_path.write_text( - json.dumps(kernel_profile_json, indent=2, sort_keys=True) - ) + kernel_profile_json_path.write_text(json.dumps(kernel_profile_json, indent=2, sort_keys=True)) # Step 7 (optional): patch the bundle's inference_perfbench_v1.json so # the renderer / aggregator picks up the kernel-profile artifact. diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_reproducer.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_reproducer.py index 507bb5f..4ed01f6 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_reproducer.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/kernel_reproducer.py @@ -12,6 +12,7 @@ scaffolder emits the canonical `linear_sm100_mpk`-shaped skeleton + the correct build flags so the boilerplate is not hand-retyped each time. """ + from __future__ import annotations from dataclasses import dataclass @@ -164,17 +165,30 @@ def scaffold_reproducer( sh = out_dir / f"build_repro_{kernel_name}.sh" dims = {"MMA_M": mma_m, "MMA_N": mma_n, "BATCH": batch, "OUT": out, "K": k} cu_text = _CU_TEMPLATE.format( - kernel_name=kernel_name, header=header, - mma_m=mma_m, mma_n=mma_n, batch=batch, out=out, k=k, + kernel_name=kernel_name, + header=header, + mma_m=mma_m, + mma_n=mma_n, + batch=batch, + out=out, + k=k, ) sh_text = _SH_TEMPLATE.format( - kernel_name=kernel_name, mirage_tree=mirage_tree, arch=arch, nvcc=nvcc, - cu_pod_path=cu_pod_path, exe_pod_path=exe_pod_path, + kernel_name=kernel_name, + mirage_tree=mirage_tree, + arch=arch, + nvcc=nvcc, + cu_pod_path=cu_pod_path, + exe_pod_path=exe_pod_path, ) if not dry_run: out_dir.mkdir(parents=True, exist_ok=True) cu.write_text(cu_text) sh.write_text(sh_text) return ScaffoldResult( - cu_path=str(cu), build_path=str(sh), kernel_name=kernel_name, dims=dims, wrote=not dry_run, + cu_path=str(cu), + build_path=str(sh), + kernel_name=kernel_name, + dims=dims, + wrote=not dry_run, ) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/lake_writer.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/lake_writer.py index 43dfe54..9e423b1 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/lake_writer.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/lake_writer.py @@ -23,14 +23,18 @@ import getpass import hashlib import json +import logging import os import re import socket import sys +from collections.abc import Callable from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Callable +from typing import Any + +import yaml from tools.perf_tune_report.capture_signature import variant_key_for from tools.perf_tune_report.helpers import resolve_operator_path @@ -41,7 +45,6 @@ read_jsonl, ) - S3_ENV_ENDPOINT = "PERFLAKE_LAKE_S3_ENDPOINT" S3_ENV_BUCKET = "PERFLAKE_LAKE_S3_BUCKET" S3_ENV_ACCESS_KEY = "PERFLAKE_LAKE_S3_ACCESS_KEY" @@ -113,6 +116,7 @@ IF_EXISTS_SKIP = "skip" IF_EXISTS_OVERWRITE = "overwrite" IF_EXISTS_CHOICES = (IF_EXISTS_FAIL, IF_EXISTS_SKIP, IF_EXISTS_OVERWRITE) +logger = logging.getLogger(__name__) class CampaignIncompleteError(RuntimeError): @@ -125,6 +129,7 @@ class CampaignIncompleteError(RuntimeError): renderer's report_status.json. """ + # Matches the campaign-dir UTC timestamp token anywhere in the name (no # anchor): handles both the canonical campaign_init layout # "-" (e.g. "20260525T081650Z-glm51-phase6", timestamp prefix) @@ -205,8 +210,7 @@ def parse_campaign_utc(campaign_id: str) -> datetime: match = _CAMPAIGN_UTC_RE.search(campaign_id) if not match: raise ValueError( - f"FATAL: campaign_id {campaign_id!r} does not end with " - f"YYYYMMDDTHHMMSSZ; cannot derive captured_at_utc." + f"FATAL: campaign_id {campaign_id!r} does not end with YYYYMMDDTHHMMSSZ; cannot derive captured_at_utc." ) return datetime( int(match["y"]), @@ -215,7 +219,7 @@ def parse_campaign_utc(campaign_id: str) -> datetime: int(match["H"]), int(match["M"]), int(match["S"]), - tzinfo=timezone.utc, + tzinfo=UTC, ) @@ -293,7 +297,10 @@ def read_render_status(campaign_dir: Path) -> RenderStatusSummary: status_path = campaign_dir / "report_status.json" if not status_path.is_file(): return RenderStatusSummary( - rendered=False, sol_complete=False, plot_ready_points=0, omitted_pages="", + rendered=False, + sol_complete=False, + plot_ready_points=0, + omitted_pages="", dcgm_grounded=False, ) data = json.loads(status_path.read_text(encoding="utf-8")) @@ -328,9 +335,9 @@ class VerdictSummary: tier: str = "draft" trials: int = 0 same_node: bool = False - decode_metric: str = "" # tpot | itl | throughput | "" + decode_metric: str = "" # tpot | itl | throughput | "" baseline_named: bool = False - latency_claim: bool = False # campaign makes a decode-latency claim + latency_claim: bool = False # campaign makes a decode-latency claim per_kernel_ref: bool = False which_kernel_claim: bool = False # campaign makes a which-kernel attribution @@ -359,14 +366,10 @@ def verdict_problems(v: VerdictSummary) -> list[str]: return [] p: list[str] = [] if v.trials < 3: - p.append( - f"verdict_tier=verdict needs >=3 trials (got {v.trials}) -- run a " - "repeated-trial controlled A/B" - ) + p.append(f"verdict_tier=verdict needs >=3 trials (got {v.trials}) -- run a repeated-trial controlled A/B") if not v.same_node: p.append( - "verdict_tier=verdict needs same_node=true (pin both arms to one node) " - "-- a cross-node delta is a DRAFT" + "verdict_tier=verdict needs same_node=true (pin both arms to one node) -- a cross-node delta is a DRAFT" ) if v.latency_claim and v.decode_metric not in ("tpot", "itl"): p.append( @@ -374,10 +377,7 @@ def verdict_problems(v: VerdictSummary) -> list[str]: f"{v.decode_metric!r}) -- output tok/s at small num_prompts is TTFT-noisy" ) if not v.baseline_named: - p.append( - "verdict_tier=verdict needs baseline_named=true (name the " - "production-representative baseline)" - ) + p.append("verdict_tier=verdict needs baseline_named=true (name the production-representative baseline)") if v.which_kernel_claim and not v.per_kernel_ref: p.append( "which-kernel verdict needs per_kernel_ref=true (nsys/ncu per-kernel " @@ -409,14 +409,12 @@ def read_next_lever(campaign_dir: Path) -> str: config_path = campaign_dir / "config.yaml" if config_path.is_file(): try: - import yaml # lazy: keep the publish path import-light - cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} nl = str(cfg.get("next_lever", "") or "").strip() if nl: return nl - except Exception: # noqa: BLE001 - a malformed config must never crash publish - pass + except (AttributeError, OSError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Ignoring malformed campaign config %s: %s", config_path, exc) return str(parse_source_md(campaign_dir / "SOURCE.md").get("next_lever", "") or "").strip() @@ -448,14 +446,12 @@ def read_close_reason(campaign_dir: Path) -> str: config_path = campaign_dir / "config.yaml" if config_path.is_file(): try: - import yaml # lazy: keep the publish path import-light - cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} cr = str(cfg.get("close_reason", "") or "").strip() if cr: return cr - except Exception: # noqa: BLE001 - a malformed config must never crash publish - pass + except (AttributeError, OSError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Ignoring malformed campaign config %s: %s", config_path, exc) return str(parse_source_md(campaign_dir / "SOURCE.md").get("close_reason", "") or "").strip() @@ -506,13 +502,7 @@ def methodology_problems(rows: list[AtlasCell]) -> list[str]: if not measured: return [] p: list[str] = [] - unlabeled = sorted( - { - r.cell_id - for r in measured - if (getattr(r, "cache_mode", "unknown") or "unknown") == "unknown" - } - ) + unlabeled = sorted({r.cell_id for r in measured if (getattr(r, "cache_mode", "unknown") or "unknown") == "unknown"}) if unlabeled: p.append( "warm/cold label missing (cache_mode=unknown) on measured cell(s): " @@ -521,19 +511,13 @@ def methodology_problems(rows: list[AtlasCell]) -> list[str]: "sweep-tail) or cold (fresh / single-shot) via the importer " "--cache-mode (AGENTS.md 'Benchmark methodology hygiene')" ) - no_shape = sorted( - { - r.cell_id - for r in measured - if int(getattr(r, "max_num_batched_tokens", 0) or 0) <= 0 - } - ) + no_shape = sorted({r.cell_id for r in measured if int(getattr(r, "max_num_batched_tokens", 0) or 0) <= 0}) if no_shape: p.append( "shape provenance missing (max_num_batched_tokens<=0) on measured " - "cell(s): " + ", ".join(no_shape) - + " -- the bench shape must be recorded, not inferred from a label" + "cell(s): " + ", ".join(no_shape) + " -- the bench shape must be recorded, not inferred from a label" ) + # ISL/OSL shape precision -- the PER-ROW half of "per-number exact shape (no smoothing)" # (added 2026-06-08, docs/METHODOLOGY.md "Full-context reporting" and AGENTS.md exact # shape"). Each measured cell carries its OWN ISL/OSL; a number is NEVER smoothed to one @@ -566,7 +550,8 @@ def _isl_osl_exempt(r: AtlasCell) -> bool: if no_isl_osl: p.append( "shape provenance missing (mean_input_tokens/mean_output_tokens<=0 on a " - "vllm-bench dataset) on measured cell(s): " + ", ".join(no_isl_osl) + "vllm-bench dataset) on measured cell(s): " + + ", ".join(no_isl_osl) + " -- record the workload ISL/OSL the number was measured at (AGENTS.md " "'Benchmark methodology hygiene'). aiperf/drive_load/aa-* " "are exempt (shape defined by the dataset name)" @@ -576,13 +561,7 @@ def _isl_osl_exempt(r: AtlasCell) -> bool: # defect without its descriptor. Flag any measured row whose str descriptor field is # still "unknown", or whose gpu_memory_utilization is unset. for fld in REQUIRED_CONTEXT_STR_FIELDS: - missing = sorted( - { - r.cell_id - for r in measured - if (str(getattr(r, fld, "unknown") or "unknown")) == "unknown" - } - ) + missing = sorted({r.cell_id for r in measured if (str(getattr(r, fld, "unknown") or "unknown")) == "unknown"}) if missing: p.append( f"full-context descriptor missing ({fld}=unknown) on measured cell(s): " @@ -590,18 +569,11 @@ def _isl_osl_exempt(r: AtlasCell) -> bool: + f" -- set {fld} via the importer/runner override or bundle metadata " "(AGENTS.md 'Benchmark methodology hygiene')" ) - no_gmu = sorted( - { - r.cell_id - for r in measured - if getattr(r, "gpu_memory_utilization", None) is None - } - ) + no_gmu = sorted({r.cell_id for r in measured if getattr(r, "gpu_memory_utilization", None) is None}) if no_gmu: p.append( "full-context descriptor missing (gpu_memory_utilization unset) on measured " - "cell(s): " + ", ".join(no_gmu) - + " -- record the gpu-memory-utilization the number was measured at" + "cell(s): " + ", ".join(no_gmu) + " -- record the gpu-memory-utilization the number was measured at" ) return p @@ -615,10 +587,10 @@ def _read_krhpa(campaign_dir: Path) -> dict[str, Any] | None: if not cfg_path.is_file(): return None try: - import yaml # lazy: keep the module import light for the publish path - data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {} - except Exception: + except (OSError, TypeError, ValueError, yaml.YAMLError): + return None + if not isinstance(data, dict): return None blk = data.get("krhpa") return blk if isinstance(blk, dict) else None @@ -630,10 +602,10 @@ def _read_krhpa_exempt_reason(campaign_dir: Path) -> str: if not cfg_path.is_file(): return "" try: - import yaml # lazy: keep the module import light for the publish path - data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {} - except Exception: + except (OSError, TypeError, ValueError, yaml.YAMLError): + return "" + if not isinstance(data, dict): return "" reason = data.get("krhpa_exempt_reason") return str(reason).strip() if reason else "" @@ -686,24 +658,20 @@ def _read_provenance(campaign_dir: Path) -> dict[str, Any] | None: cfg_path = campaign_dir / "config.yaml" if cfg_path.is_file(): try: - import yaml # lazy: keep the publish path import-light - data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {} blk = data.get("provenance") if isinstance(blk, dict): return blk - except Exception: - pass + except (AttributeError, OSError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Ignoring malformed provenance config %s: %s", cfg_path, exc) js = campaign_dir / "provenance.json" if js.is_file(): try: - import json - blk = json.loads(js.read_text(encoding="utf-8")) if isinstance(blk, dict): return blk - except Exception: - pass + except (OSError, TypeError, ValueError) as exc: + logger.debug("Ignoring malformed provenance sidecar %s: %s", js, exc) return None @@ -768,9 +736,7 @@ def read_roofline_gap_arms(campaign_dir: Path) -> dict[str, str]: return {} -def per_arm_roofline_problems( - render_status: RenderStatusSummary, campaign_dir: Path -) -> list[str]: +def per_arm_roofline_problems(render_status: RenderStatusSummary, campaign_dir: Path) -> list[str]: """Per-arm roofline gate. ``roofline_problems`` above refuses only when page 7 is ENTIRELY absent (campaign-level "no arm has a roofline"). This refuses when SOME arm lacks a roofline -- the "baseline + EACH variant @@ -1018,18 +984,15 @@ def build_atlas_table(rows: list[AtlasCell], campaign_id: str) -> Any: _vk = "" columns["variant_key"].append(_vk) columns["backend"].append(row.backend or "") - columns["serving_engine"].append( - getattr(row, "serving_engine", "") or engine_for_backend(row.backend or "") - ) + columns["serving_engine"].append(getattr(row, "serving_engine", "") or engine_for_backend(row.backend or "")) # Ledger-to-atlas data-capture gaps (added 2026-06-07). columns["router_policy"].append(getattr(row, "router_policy", "") or "") columns["prefix_reuse"].append(getattr(row, "prefix_reuse", None)) columns["per_replica_cache_hit"].append(getattr(row, "per_replica_cache_hit", None)) columns["acceptance_length"].append(getattr(row, "acceptance_length", None)) columns["spec_accept_rate"].append(getattr(row, "spec_accept_rate", None)) - columns["kv_cache_tokens"].append( - int(row.kv_cache_tokens) if getattr(row, "kv_cache_tokens", None) is not None else None - ) + kv_cache_tokens = row.kv_cache_tokens + columns["kv_cache_tokens"].append(int(kv_cache_tokens) if kv_cache_tokens is not None else None) columns["ep_mode"].append(getattr(row, "ep_mode", "") or "") columns["dcgm_sm_active"].append(getattr(row, "dcgm_sm_active", None)) columns["dcgm_dram_active"].append(getattr(row, "dcgm_dram_active", None)) @@ -1234,7 +1197,7 @@ def build_campaign_row( ] ) champion = _read_champion_select(campaign_dir) or {} - published_at = published_at_utc or datetime.now(timezone.utc) + published_at = published_at_utc or datetime.now(UTC) publisher_op = publisher_operator or os.environ.get(PUBLISHER_ENV) or getpass.getuser() publisher_h = publisher_host or socket.gethostname() @@ -1299,7 +1262,7 @@ def build_campaign_row( return pa.table(columns, schema=schema) -def _walk_cell_artifacts(campaign_dir: Path, filename: str) -> "list[tuple[str, Path, dict[str, Any]]]": +def _walk_cell_artifacts(campaign_dir: Path, filename: str) -> list[tuple[str, Path, dict[str, Any]]]: """Walk ``/cells/*/`` and load each JSON. Returns a list of ``(cell_id, path, payload)`` ordered by cell dir name. @@ -1325,7 +1288,7 @@ def _walk_cell_artifacts(campaign_dir: Path, filename: str) -> "list[tuple[str, return out -def _resolve_sol_ceilings(campaign_dir: Path, rows: list[AtlasCell]) -> "tuple[dict[str, Any] | None, str | None, str]": +def _resolve_sol_ceilings(campaign_dir: Path, rows: list[AtlasCell]) -> tuple[dict[str, Any] | None, str | None, str]: """Find + load sol-ceilings.yaml and pick the hardware key. Mirrors ``render_report.discover_sol_inputs`` but without importing the @@ -1347,8 +1310,7 @@ def _resolve_sol_ceilings(campaign_dir: Path, rows: list[AtlasCell]) -> "tuple[d # Canonical bundle name first, then a name-agnostic bundle-root fallback # (/configs/sol-ceilings.yaml) so a future submodule rename needs # no code edit; the campaign's own bundle root is reached first. - relcands = (Path("perf-tune-report") / "configs" / "sol-ceilings.yaml", - Path("configs") / "sol-ceilings.yaml") + relcands = (Path("perf-tune-report") / "configs" / "sol-ceilings.yaml", Path("configs") / "sol-ceilings.yaml") for parent in [cur, *cur.parents]: for rel in relcands: cand = parent / rel @@ -1446,6 +1408,7 @@ def emit(**kw: Any) -> None: # L1 -- zymtrace per-category sample-share + ceiling from tools.perf_tune_report.renderer import sol_roofline # stdlib-only + for cell_id, path, payload in _walk_cell_artifacts(campaign_dir, "kernels.json"): sha = _sha256_of(path) if ceilings is not None and hw_key is not None: @@ -1453,15 +1416,25 @@ def emit(**kw: Any) -> None: cat_map = ceilings.get("category_ceiling_map", {}) for r in sol_roofline.compute_category_sol(payload, hw_data, cat_map): emit( - cell_id=cell_id, category=r["category"], sol_level="L1", - gpu_time_share_pct=r["time_share_pct"], pct_sol=None, - bound=r["bound"], ceiling_key=r["ceiling_metric"], - ceiling_value=r["ceiling_value"], ceiling_units=r["ceiling_units"], - measured_value=None, measured_units=None, - attributed_bytes_total=None, attributed_flops_total=None, - arithmetic_intensity_flops_per_byte=None, kernel_name=None, - hw_key=hw_key, source_artifact="kernels.json", - source_artifact_sha256=sha, sol_ceilings_yaml_sha256=yaml_sha, + cell_id=cell_id, + category=r["category"], + sol_level="L1", + gpu_time_share_pct=r["time_share_pct"], + pct_sol=None, + bound=r["bound"], + ceiling_key=r["ceiling_metric"], + ceiling_value=r["ceiling_value"], + ceiling_units=r["ceiling_units"], + measured_value=None, + measured_units=None, + attributed_bytes_total=None, + attributed_flops_total=None, + arithmetic_intensity_flops_per_byte=None, + kernel_name=None, + hw_key=hw_key, + source_artifact="kernels.json", + source_artifact_sha256=sha, + sol_ceilings_yaml_sha256=yaml_sha, ) else: # No ceilings resolvable: still record per-category time-share. @@ -1471,14 +1444,25 @@ def emit(**kw: Any) -> None: if int(samples) == 0: continue emit( - cell_id=cell_id, category=cat, sol_level="L1", - gpu_time_share_pct=100.0 * int(samples) / total, pct_sol=None, - bound=None, ceiling_key=None, ceiling_value=None, ceiling_units=None, - measured_value=None, measured_units=None, - attributed_bytes_total=None, attributed_flops_total=None, - arithmetic_intensity_flops_per_byte=None, kernel_name=None, - hw_key=hw_key, source_artifact="kernels.json", - source_artifact_sha256=sha, sol_ceilings_yaml_sha256=yaml_sha, + cell_id=cell_id, + category=cat, + sol_level="L1", + gpu_time_share_pct=100.0 * int(samples) / total, + pct_sol=None, + bound=None, + ceiling_key=None, + ceiling_value=None, + ceiling_units=None, + measured_value=None, + measured_units=None, + attributed_bytes_total=None, + attributed_flops_total=None, + arithmetic_intensity_flops_per_byte=None, + kernel_name=None, + hw_key=hw_key, + source_artifact="kernels.json", + source_artifact_sha256=sha, + sol_ceilings_yaml_sha256=yaml_sha, ) # L2 + L3 -- DCGM cross-attribution + workload resources @@ -1495,16 +1479,25 @@ def emit(**kw: Any) -> None: pct = c.get("sol_pct_bw") if c.get("sol_pct_bw") is not None else c.get("sol_pct_compute") meas = munits = None emit( - cell_id=cell_id, category=c.get("category"), sol_level="L2", - gpu_time_share_pct=c.get("time_share_pct"), pct_sol=pct, - bound=bound, ceiling_key=c.get("ceiling_metric"), - ceiling_value=None, ceiling_units=None, - measured_value=meas, measured_units=munits, + cell_id=cell_id, + category=c.get("category"), + sol_level="L2", + gpu_time_share_pct=c.get("time_share_pct"), + pct_sol=pct, + bound=bound, + ceiling_key=c.get("ceiling_metric"), + ceiling_value=None, + ceiling_units=None, + measured_value=meas, + measured_units=munits, attributed_bytes_total=c.get("attributed_bytes_total"), attributed_flops_total=c.get("attributed_flops_total"), - arithmetic_intensity_flops_per_byte=None, kernel_name=None, - hw_key=cell_hw, source_artifact="dcgm_correlation.json", - source_artifact_sha256=sha, sol_ceilings_yaml_sha256=yaml_sha, + arithmetic_intensity_flops_per_byte=None, + kernel_name=None, + hw_key=cell_hw, + source_artifact="dcgm_correlation.json", + source_artifact_sha256=sha, + sol_ceilings_yaml_sha256=yaml_sha, ) for res in payload.get("resources", []) or []: bytes_ps, tflops = res.get("measured_bytes_per_s"), res.get("measured_tflops_avg") @@ -1515,16 +1508,25 @@ def emit(**kw: Any) -> None: else: meas, munits = None, None emit( - cell_id=cell_id, category=res.get("peak_key"), sol_level="L3", - gpu_time_share_pct=None, pct_sol=res.get("sol_pct"), - bound=None, ceiling_key=res.get("peak_key"), - ceiling_value=res.get("peak_per_gpu"), ceiling_units=res.get("peak_per_gpu_units"), - measured_value=meas, measured_units=munits, + cell_id=cell_id, + category=res.get("peak_key"), + sol_level="L3", + gpu_time_share_pct=None, + pct_sol=res.get("sol_pct"), + bound=None, + ceiling_key=res.get("peak_key"), + ceiling_value=res.get("peak_per_gpu"), + ceiling_units=res.get("peak_per_gpu_units"), + measured_value=meas, + measured_units=munits, attributed_bytes_total=res.get("measured_bytes_total"), attributed_flops_total=None, - arithmetic_intensity_flops_per_byte=None, kernel_name=None, - hw_key=cell_hw, source_artifact="dcgm_correlation.json", - source_artifact_sha256=sha, sol_ceilings_yaml_sha256=yaml_sha, + arithmetic_intensity_flops_per_byte=None, + kernel_name=None, + hw_key=cell_hw, + source_artifact="dcgm_correlation.json", + source_artifact_sha256=sha, + sol_ceilings_yaml_sha256=yaml_sha, ) # L4 -- ncu per-kernel arithmetic intensity @@ -1536,16 +1538,25 @@ def emit(**kw: Any) -> None: if pct is None: pct = k.get("achieved_sm_pct_peak") emit( - cell_id=cell_id, category=k.get("category") or "Other", sol_level="L4", - gpu_time_share_pct=None, pct_sol=pct, bound=None, - ceiling_key=None, ceiling_value=None, ceiling_units=None, - measured_value=k.get("achieved_tflops"), measured_units="TFLOPS", + cell_id=cell_id, + category=k.get("category") or "Other", + sol_level="L4", + gpu_time_share_pct=None, + pct_sol=pct, + bound=None, + ceiling_key=None, + ceiling_value=None, + ceiling_units=None, + measured_value=k.get("achieved_tflops"), + measured_units="TFLOPS", attributed_bytes_total=k.get("dram_bytes_total"), attributed_flops_total=k.get("sm_flops_total"), arithmetic_intensity_flops_per_byte=k.get("arithmetic_intensity_flops_per_byte"), kernel_name=k.get("name"), - hw_key=cell_hw, source_artifact="ncu_kernels.json", - source_artifact_sha256=sha, sol_ceilings_yaml_sha256=yaml_sha, + hw_key=cell_hw, + source_artifact="ncu_kernels.json", + source_artifact_sha256=sha, + sol_ceilings_yaml_sha256=yaml_sha, ) return pa.table(columns, schema=schema) @@ -1720,11 +1731,7 @@ def build_cost_table( if point is None: continue power = _cell_power_watts_per_gpu(campaign_dir, point.cell_id) - tokens_per_watt = ( - point.output_tps_per_gpu / power - if power is not None and power > 0 - else None - ) + tokens_per_watt = point.output_tps_per_gpu / power if power is not None and power > 0 else None columns["campaign_id"].append(campaign_id) columns["model"].append(g.model) columns["hardware"].append(g.hardware) @@ -1760,9 +1767,7 @@ def _quality_metrics(extra: dict[str, Any]) -> dict[str, float]: canonical = extra.get("quality_metrics") if isinstance(canonical, dict): return { - str(k): float(v) - for k, v in canonical.items() - if isinstance(v, (int, float)) and not isinstance(v, bool) + str(k): float(v) for k, v in canonical.items() if isinstance(v, (int, float)) and not isinstance(v, bool) } out: dict[str, float] = {} for k, v in extra.items(): @@ -1920,7 +1925,7 @@ def build_champion_table( _HW_TOKEN_TO_KEY = {"B200": "b200_sm100", "GB300": "gb300_nvl72", "H100": "h100_sxm"} -def _peak_for(ceilings: dict[str, Any] | None, hw_token: str, quant: str) -> "tuple[float | None, float | None]": +def _peak_for(ceilings: dict[str, Any] | None, hw_token: str, quant: str) -> tuple[float | None, float | None]: """(compute_peak_pflops_per_gpu, hbm_peak_tbps_per_gpu) from sol-ceilings, keyed by the roofline payload's bare hardware token + quant. (None, None) when ceilings are unavailable -> the %-of-peak columns are left null.""" @@ -1970,8 +1975,8 @@ def build_roofline_table( pa.field("quant", pa.string(), nullable=False), pa.field("kv_dtype", pa.string(), nullable=False), pa.field("tensor_parallel", pa.int64(), nullable=False), - pa.field("phase", pa.string(), nullable=False), # decode | prefill - pa.field("concurrency", pa.int64(), nullable=True), # decode C (null for prefill) + pa.field("phase", pa.string(), nullable=False), # decode | prefill + pa.field("concurrency", pa.int64(), nullable=True), # decode C (null for prefill) pa.field("isl", pa.int64(), nullable=True), pa.field("osl", pa.int64(), nullable=True), pa.field("median_tpot_ms", pa.float64(), nullable=True), @@ -1981,9 +1986,9 @@ def build_roofline_table( pa.field("arithmetic_intensity", pa.float64(), nullable=True), pa.field("achieved_tflops_per_gpu", pa.float64(), nullable=True), pa.field("hbm_delivered_Bps_per_gpu", pa.float64(), nullable=True), - pa.field("hbm_delivered_pct", pa.float64(), nullable=True), # delivered / peak * 100 + pa.field("hbm_delivered_pct", pa.float64(), nullable=True), # delivered / peak * 100 # measured DCGM active fractions (the utilization-vs-C curves) - pa.field("dram_active_pct", pa.float64(), nullable=True), # DRAM_ACTIVE * 100 (proxy) + pa.field("dram_active_pct", pa.float64(), nullable=True), # DRAM_ACTIVE * 100 (proxy) pa.field("tensor_active_pct", pa.float64(), nullable=True), pa.field("sm_active_pct", pa.float64(), nullable=True), # per-GPU ceilings + ridge (so the roofline line is queryable) @@ -1996,13 +2001,13 @@ def build_roofline_table( ) from tools.perf_tune_report import roofline_math # pure module (no matplotlib) - def _shape_for(payload: dict[str, Any]) -> "roofline_math.ModelShape | None": + def _shape_for(payload: dict[str, Any]) -> roofline_math.ModelShape | None: emb = payload.get("analytical_shape") if isinstance(emb, dict) and emb.get("hidden_size"): try: return roofline_math.shape_from_dict(emb) - except Exception: # noqa: BLE001 - pass + except TypeError: + return roofline_math.shape_for_model(payload.get("model", "")) return roofline_math.shape_for_model(payload.get("model", "")) def _coords(pt: dict[str, Any], phase: str, shape, quant: str, tp: int, kvd: str): @@ -2020,8 +2025,7 @@ def _coords(pt: dict[str, Any], phase: str, shape, quant: str, tp: int, kvd: str return None, None, None ctx = int((pt.get("isl") or 256) + (pt.get("osl") or 512) // 2) ai = shape.decode_arithmetic_intensity(c, ctx, quant, kvd) - union = (min(shape.n_routed_experts, shape.n_experts_per_tok * c) - if shape.is_moe else 0) + union = min(shape.n_routed_experts, shape.n_experts_per_tok * c) if shape.is_moe else 0 bpt = shape.active_weight_bytes(union, quant) / c + shape.kv_bytes_per_token(ctx, kvd) else: isl, inp, dur = pt.get("isl"), pt.get("total_input_tokens"), pt.get("duration") @@ -2105,10 +2109,7 @@ def s3_key_for(table_name: str, campaign_id: str, captured_at: datetime) -> str: key. The downstream Spark MERGE INTO can then key on campaign_id. """ dt = captured_at.strftime("%Y-%m-%d") - return ( - f"{S3_PREFIX}/{table_name}/" - f"dt={dt}/campaign={campaign_id}/part-0.parquet" - ) + return f"{S3_PREFIX}/{table_name}/dt={dt}/campaign={campaign_id}/part-0.parquet" def _make_s3_client(cfg: S3Config) -> Any: @@ -2193,7 +2194,7 @@ def upload_to_s3( try: client.head_object(Bucket=bucket, Key=key) exists = True - except Exception as error: # noqa: BLE001 - boto3 exposes service errors at runtime + except Exception as error: if not _is_s3_not_found(error): raise @@ -2209,7 +2210,7 @@ def upload_to_s3( try: with local_path.open("rb") as body: client.put_object(Body=body, **put_kwargs) - except Exception as error: # noqa: BLE001 - boto3 exposes service errors at runtime + except Exception as error: if if_exists == IF_EXISTS_FAIL and _is_s3_precondition_failed(error): raise _object_exists_error(bucket, key) from error raise @@ -2296,9 +2297,7 @@ def publish( Returns a ``PublishResult`` with per-table outcomes. """ if if_exists not in IF_EXISTS_CHOICES: - raise ValueError( - f"if_exists must be one of {IF_EXISTS_CHOICES}; got {if_exists!r}" - ) + raise ValueError(f"if_exists must be one of {IF_EXISTS_CHOICES}; got {if_exists!r}") campaign_dir = campaign_dir.resolve() if not campaign_dir.is_dir(): @@ -2355,8 +2354,7 @@ def publish( if problems: if strict: raise CampaignIncompleteError( - f"--strict: refusing to publish incomplete campaign " - f"{campaign_id!r}: " + "; ".join(problems) + f"--strict: refusing to publish incomplete campaign {campaign_id!r}: " + "; ".join(problems) ) print( f"WARNING: publishing campaign {campaign_id!r} with recorded gaps " @@ -2399,7 +2397,7 @@ def publish( file=sys.stderr, ) - published_at = published_at_utc or datetime.now(timezone.utc) + published_at = published_at_utc or datetime.now(UTC) atlas_table = build_atlas_table(rows, campaign_id) campaign_table = build_campaign_row( campaign_dir, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/campaign_run.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/campaign_run.py index 9904c5b..90531b3 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/campaign_run.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/campaign_run.py @@ -50,10 +50,11 @@ import json import time +from collections.abc import Callable, Sequence from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Callable, Sequence +from typing import Any # Backends that target an already-running endpoint URL: they do NOT manage a # helm release, so run_one_cell skips the helm_upgrade + warmup steps for them. @@ -84,14 +85,9 @@ class CellPlan: def __post_init__(self) -> None: if not self.id or not self.id.replace("-", "").replace("_", "").isalnum(): - raise ValueError( - f"CellPlan.id must be alphanumeric (+ - or _), got {self.id!r}" - ) + raise ValueError(f"CellPlan.id must be alphanumeric (+ - or _), got {self.id!r}") if self.backend not in ("vllm-sweep", "aiperf", "aa", "trtllm"): - raise ValueError( - f"CellPlan.backend must be one of vllm-sweep|aiperf|aa|trtllm; " - f"got {self.backend!r}" - ) + raise ValueError(f"CellPlan.backend must be one of vllm-sweep|aiperf|aa|trtllm; got {self.backend!r}") if not self.concurrencies: raise ValueError(f"CellPlan {self.id}: concurrencies must be non-empty") @@ -138,8 +134,6 @@ def to_dict(self) -> dict[str, Any]: d["campaign_dir"] = str(self.campaign_dir) d["pdf_path"] = str(self.pdf_path) if self.pdf_path else None d["atlas_path"] = str(self.atlas_path) if self.atlas_path else None - for c in d["per_cell"]: - pass # already dict-friendly via asdict return d @@ -184,7 +178,7 @@ class StepFns: def _utc_now_iso() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") def _verdict_rollup(per_cell: Sequence[CellStepResult]) -> str: @@ -257,14 +251,9 @@ def run_one_cell( # do not manage a helm release, so the deploy step is a no-op pass. if endpoint_only: helm_ok = True - notes.append( - f"backend={cell.backend} targets an existing endpoint; " - "skipping helm_upgrade" - ) + notes.append(f"backend={cell.backend} targets an existing endpoint; skipping helm_upgrade") else: - helm_ok = step_fns.helm_upgrade( - target_namespace, target_release, chart_dir, cell.helm_overrides - ) + helm_ok = step_fns.helm_upgrade(target_namespace, target_release, chart_dir, cell.helm_overrides) if not helm_ok: notes.append("helm_upgrade returned False; aborting cell") aborted_early = True @@ -293,9 +282,7 @@ def run_one_cell( notes.append("zymtrace returned False; not blocking cell") # Step 6: import_perf_bench (idempotent — converts raw to normalized). - import_ok = step_fns.import_bundle( - campaign_dir / "cells" / cell.id, campaign_dir, cell.id - ) + import_ok = step_fns.import_bundle(campaign_dir / "cells" / cell.id, campaign_dir, cell.id) # Step 7: atlas_aggregate (campaign-level rollup). aggregate_ok = step_fns.aggregate(campaign_dir) @@ -321,9 +308,7 @@ def run_one_cell( # Step 8: report_render (re-render PDF after each cell — important # for the safe-Ctrl-C contract). - render_ok = step_fns.render( - campaign_dir, campaign_dir / f"{cell.id}-report.pdf" - ) + render_ok = step_fns.render(campaign_dir, campaign_dir / f"{cell.id}-report.pdf") # Step 9: baseline_record. baseline_record_ok = step_fns.baseline_record(campaign_dir, cell.id) @@ -337,9 +322,7 @@ def run_one_cell( ) verdict = "RED" else: - verdict = step_fns.baseline_diff( - campaign_dir, cell.id, comparator_baseline - ) + verdict = step_fns.baseline_diff(campaign_dir, cell.id, comparator_baseline) finally: # CRITICAL: always-resume on Slurm-on-K8s drain. This runs even on Ctrl-C @@ -354,24 +337,47 @@ def run_one_cell( "Slurm-on-K8s partition may be left in DRAIN state — operator must " "manually `scontrol update state=RESUME`!" ) - except Exception as exc: + except (OSError, RuntimeError, ValueError) as exc: notes.append(f"resume raised: {type(exc).__name__}: {exc}") resume_ok = False else: resume_ok = True return _make_cell_result( - cell, started, started_iso, drain_ok, helm_ok, - warmup_ok, bench_ok, zymtrace_ok, import_ok, - aggregate_ok, render_ok, baseline_record_ok, - verdict, resume_ok, notes, + cell, + started, + started_iso, + drain_ok, + helm_ok, + warmup_ok, + bench_ok, + zymtrace_ok, + import_ok, + aggregate_ok, + render_ok, + baseline_record_ok, + verdict, + resume_ok, + notes, ) def _make_cell_result( - cell, started, started_iso, drain_ok, helm_ok, warmup_ok, bench_ok, - zymtrace_ok, import_ok, aggregate_ok, render_ok, baseline_record_ok, - verdict, resume_ok, notes, + cell, + started, + started_iso, + drain_ok, + helm_ok, + warmup_ok, + bench_ok, + zymtrace_ok, + import_ok, + aggregate_ok, + render_ok, + baseline_record_ok, + verdict, + resume_ok, + notes, ) -> CellStepResult: return CellStepResult( cell_id=cell.id, @@ -448,12 +454,8 @@ def run_campaign( aborted_at = cell.id break - cells_completed = sum( - 1 for r in per_cell if r.bench_ok and r.import_ok and r.aggregate_ok - ) - cells_failed = sum( - 1 for r in per_cell if not r.bench_ok or not r.import_ok - ) + cells_completed = sum(1 for r in per_cell if r.bench_ok and r.import_ok and r.aggregate_ok) + cells_failed = sum(1 for r in per_cell if not r.bench_ok or not r.import_ok) return CampaignRunResult( campaign_dir=campaign_dir, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/production_steps.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/production_steps.py index d3ceb56..763d56b 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/production_steps.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/orchestrator/production_steps.py @@ -40,20 +40,28 @@ import contextlib import io import json +import logging import math import subprocess import time +from collections.abc import Sequence from pathlib import Path -from typing import Any, Sequence +from typing import Any + +import yaml from tools.perf_tune_report.helpers import resolve_operator_path +from tools.perf_tune_report.importers.zymtrace_kernels import ( + ZymtraceTSVMalformed, + ZymtraceTSVMissing, +) from tools.perf_tune_report.orchestrator.campaign_run import CellPlan, StepFns - _CMD_TIMEOUT_S = 120 _WARMUP_TIMEOUT_S = 60 _WARMUP_POLL_INTERVAL_S = 5 _SERVER_ROOT = Path(__file__).resolve().parents[3] +logger = logging.getLogger(__name__) _BASELINE_METRICS: dict[str, tuple[str, str, str]] = { "output_tps_per_gpu": ("higher-is-better", "max", "tokens/s/GPU"), @@ -72,9 +80,7 @@ def _run_cmd(cmd: list[str], *, timeout: int = _CMD_TIMEOUT_S) -> bool: """Run a command, returning True on exit 0, False on any failure.""" try: - result = subprocess.run( - cmd, capture_output=True, text=True, timeout=timeout, check=False - ) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, check=False) return result.returncode == 0 except subprocess.TimeoutExpired: return False @@ -118,8 +124,13 @@ def step_helm_upgrade( ``overrides["set"]``. """ cmd = [ - "helm", "upgrade", "--install", release, str(chart_dir), - "--namespace", namespace, + "helm", + "upgrade", + "--install", + release, + str(chart_dir), + "--namespace", + namespace, ] if "path" in overrides: cmd.extend(["-f", str(overrides["path"])]) @@ -133,9 +144,15 @@ def step_warmup(namespace: str, release: str, cell_id: str) -> bool: deadline = time.monotonic() + _WARMUP_TIMEOUT_S while time.monotonic() < deadline: cmd = [ - "kubectl", "-n", namespace, - "exec", f"deploy/{release}", - "--", "curl", "-sf", "http://localhost:8000/v1/models", + "kubectl", + "-n", + namespace, + "exec", + f"deploy/{release}", + "--", + "curl", + "-sf", + "http://localhost:8000/v1/models", ] if _run_cmd(cmd, timeout=10): return True @@ -184,28 +201,32 @@ def step_bench(campaign_dir: Path, cell: CellPlan) -> bool: try: if cell.backend == "vllm-sweep": from tools.perf_tune_report.runners.vllm_sweep import run_cell as run_vllm + cfg = _cell_config_from_plan(cell) - serve_cmd = cell.backend_config.get( - "serve_cmd", cell.helm_overrides.get("serve_cmd", "") - ) - bench_cmd = cell.backend_config.get( - "bench_cmd", cell.helm_overrides.get("bench_cmd", "") - ) + serve_cmd = cell.backend_config.get("serve_cmd", cell.helm_overrides.get("serve_cmd", "")) + bench_cmd = cell.backend_config.get("bench_cmd", cell.helm_overrides.get("bench_cmd", "")) if not serve_cmd or not bench_cmd: return False res = run_vllm( - cfg, campaign_dir, - serve_cmd=serve_cmd, bench_cmd=bench_cmd, dry_run=False, + cfg, + campaign_dir, + serve_cmd=serve_cmd, + bench_cmd=bench_cmd, + dry_run=False, ) return res.row_count > 0 if cell.backend == "aiperf": # Replay mooncake-trace replay against an existing endpoint URL. # Config comes from the cell's ``aiperf:`` block (cell.backend_config). from tools.perf_tune_report.runners.aiperf_bench import run_cell as run_aiperf + cfgd = cell.backend_config required = ( - "namespace", "bench_pod", "kube_context", - "endpoint_url", "served_model", + "namespace", + "bench_pod", + "kube_context", + "endpoint_url", + "served_model", ) if any(not cfgd.get(k) for k in required): return False @@ -226,7 +247,9 @@ def step_bench(campaign_dir: Path, cell: CellPlan) -> bool: # Artificial-Analysis synthetic fixed-shape (or dataset-replay) via # AIPerf against an endpoint URL. Config from the ``aa:`` block. import os + from tools.perf_tune_report.runners.aa_bench import run_cell as run_aa + cfgd = cell.backend_config model = cfgd.get("model") url = cfgd.get("url") @@ -253,9 +276,7 @@ def step_bench(campaign_dir: Path, cell: CellPlan) -> bool: endpoint_type=cfgd.get("endpoint_type", "chat"), api_key=api_key, tokenizer=cfgd.get("tokenizer"), - tokenizer_trust_remote_code=bool( - cfgd.get("tokenizer_trust_remote_code", True) - ), + tokenizer_trust_remote_code=bool(cfgd.get("tokenizer_trust_remote_code", True)), mode=cfgd.get("mode", "synthetic"), request_count=int(cfgd.get("request_count", 10)), custom_dataset_type=cfgd.get("custom_dataset_type", "mooncake_trace"), @@ -269,7 +290,17 @@ def step_bench(campaign_dir: Path, cell: CellPlan) -> bool: return res.row_count > 0 if cell.backend == "trtllm": return False - except Exception: + except ( + ImportError, + KeyError, + NotImplementedError, + OSError, + RuntimeError, + subprocess.SubprocessError, + TypeError, + ValueError, + ) as exc: + logger.debug("Benchmark step failed for cell %s: %s", cell.id, exc) return False return False @@ -291,6 +322,7 @@ def step_import(cell_dir: Path, campaign_dir: Path, cell_id: str) -> bool: """Wrap ``import_bundle_auto`` so it adheres to the StepFns signature.""" try: from tools.perf_tune_report.importers import import_bundle_auto + # The cell_dir IS the bundle dir under the new pipeline (where the # bench output lands). For backward compat, accept either layout: # cell_dir/raw or cell_dir/bench-c. @@ -301,16 +333,19 @@ def step_import(cell_dir: Path, campaign_dir: Path, cell_id: str) -> bool: dry_run=False, ) return True - except Exception: + except (ImportError, OSError, TypeError, ValueError, ZymtraceTSVMalformed, ZymtraceTSVMissing) as exc: + logger.debug("Import step failed for cell %s: %s", cell_id, exc) return False def step_aggregate(campaign_dir: Path) -> bool: try: from tools.perf_tune_report.aggregator import aggregate + aggregate(campaign_dir) return True - except Exception: + except (ImportError, OSError, TypeError, ValueError) as exc: + logger.debug("Aggregate step failed for %s: %s", campaign_dir, exc) return False @@ -345,13 +380,16 @@ def step_dcgm_correlate(campaign_dir: Path, cell_id: str) -> bool: YAML -> returns False (loud-skip via the caller's note), does NOT abort. """ try: - import yaml as _yaml - from tools.perf_tune_report.dcgm_correlate import ( + FrozenYamlMalformed, correlate_from_frozen, write_correlation, ) + except ImportError as exc: + logger.debug("DCGM correlation imports failed for cell %s: %s", cell_id, exc) + return False + try: cell_dir = campaign_dir / "cells" / cell_id frozen = cell_dir / "dcgm-frozen.yaml" if not frozen.is_file(): @@ -359,7 +397,7 @@ def step_dcgm_correlate(campaign_dir: Path, cell_id: str) -> bool: ceilings_path = _discover_ceilings_yaml(campaign_dir) if ceilings_path is None: return False - ceilings = _yaml.safe_load(ceilings_path.read_text()) + ceilings = yaml.safe_load(ceilings_path.read_text()) kernels = cell_dir / "kernels.json" result = correlate_from_frozen( frozen, @@ -369,19 +407,56 @@ def step_dcgm_correlate(campaign_dir: Path, cell_id: str) -> bool: ) write_correlation(result, cell_dir) return True - except Exception: + except ( + FrozenYamlMalformed, + KeyError, + OSError, + TypeError, + ValueError, + yaml.YAMLError, + ) as exc: + logger.debug("DCGM correlation step failed for cell %s: %s", cell_id, exc) return False def step_render(campaign_dir: Path, out_pdf: Path) -> bool: try: - from tools.perf_tune_report.renderer.render_report import render_report + from tools.perf_tune_report.renderer.champion_select import ( + ChampionSelectJsonMalformed, + ) + from tools.perf_tune_report.renderer.prefill_decode_roofline import ( + RooflineSweepJsonMalformed, + ) + from tools.perf_tune_report.renderer.render_report import ( + DcgmCorrelationJsonMalformed, + KernelsJsonMalformed, + NcuKernelsJsonMalformed, + render_report, + ) + from tools.perf_tune_report.renderer.sol_roofline import SoLCeilingsMalformed + except ImportError as exc: + logger.debug("Render imports failed for %s: %s", campaign_dir, exc) + return False + + try: atlas = campaign_dir / "atlas.jsonl" if not atlas.is_file(): return False render_report(atlas, out_pdf, title=f"campaign {campaign_dir.name}") return True - except Exception: + except ( + ChampionSelectJsonMalformed, + DcgmCorrelationJsonMalformed, + KernelsJsonMalformed, + NcuKernelsJsonMalformed, + OSError, + RooflineSweepJsonMalformed, + RuntimeError, + SoLCeilingsMalformed, + TypeError, + ValueError, + ) as exc: + logger.debug("Render step failed for %s: %s", campaign_dir, exc) return False @@ -390,17 +465,11 @@ def _baseline_repo_root() -> Path: import os override = os.environ.get("PROFILE_AND_OPTIMIZE_REPO_ROOT") - return ( - resolve_operator_path(override, label="PROFILE_AND_OPTIMIZE_REPO_ROOT") - if override - else _SERVER_ROOT - ) + return resolve_operator_path(override, label="PROFILE_AND_OPTIMIZE_REPO_ROOT") if override else _SERVER_ROOT def _cell_headline(campaign_dir: Path, cell_id: str) -> dict[str, Any] | None: """Materialize one comparable scalar headline for a campaign cell.""" - import yaml - config_path = campaign_dir / "config.yaml" atlas_path = campaign_dir / "atlas.jsonl" if not config_path.is_file() or not atlas_path.is_file(): @@ -423,9 +492,7 @@ def _cell_headline(campaign_dir: Path, cell_id: str) -> dict[str, Any] | None: if metric not in _BASELINE_METRICS: return None direction, reduction, unit = _BASELINE_METRICS[metric] - requested_concurrency = campaign_config.get( - "baseline_concurrency", config.get("baseline_concurrency") - ) + requested_concurrency = campaign_config.get("baseline_concurrency", config.get("baseline_concurrency")) candidates: list[tuple[float, int | None]] = [] try: @@ -477,6 +544,7 @@ def step_baseline_record(campaign_dir: Path, cell_id: str) -> bool: """Register one directional scalar headline for the campaign cell.""" try: from tools.perf_baseline.perf_baseline_cli import cmd_record + headline = _cell_headline(campaign_dir, cell_id) if headline is None: return False @@ -496,7 +564,8 @@ def step_baseline_record(campaign_dir: Path, cell_id: str) -> bool: return cmd_record(ns) == 0 except SystemExit: return False - except Exception: + except (ImportError, KeyError, OSError, RuntimeError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Baseline record step failed for cell %s: %s", cell_id, exc) return False @@ -513,6 +582,7 @@ def step_baseline_diff(campaign_dir: Path, cell_id: str, comparator: str) -> str return "RED" try: from tools.perf_baseline.perf_baseline_cli import cmd_diff + headline = _cell_headline(campaign_dir, cell_id) if headline is None: return "RED" @@ -540,7 +610,8 @@ def step_baseline_diff(campaign_dir: Path, cell_id: str, comparator: str) -> str return verdict if verdict in {"GREEN", "YELLOW", "RED"} else "RED" except SystemExit: return "RED" - except Exception: + except (ImportError, KeyError, OSError, RuntimeError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Baseline diff step failed for cell %s: %s", cell_id, exc) return "RED" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/perf_tune_report_cli.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/perf_tune_report_cli.py index 098e5ea..58c0586 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/perf_tune_report_cli.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/perf_tune_report_cli.py @@ -34,14 +34,25 @@ import argparse import json +import logging import os import shutil import sys from pathlib import Path from typing import Any +from tools.perf_tune_report import champion_select as champion_select_lib from tools.perf_tune_report.aggregator import aggregate +from tools.perf_tune_report.capture_signature import ( + build_plan as build_capture_plan, +) +from tools.perf_tune_report.capture_signature import ( + materialize_reuse, +) from tools.perf_tune_report.coverage import summarize +from tools.perf_tune_report.graph_diff import ( + diff_graph_logs, +) from tools.perf_tune_report.helpers import ( emit, load_yaml, @@ -54,31 +65,25 @@ synthetic_fixture_path, utc_timestamp_slug, ) -from tools.perf_tune_report.importers import import_bundle_auto, import_perf_bench_bundle -from tools.perf_tune_report.importers import import_roofline_sweep_bundle -from tools.perf_tune_report.importers import detect_variant_ab, import_variant_ab_bundle +from tools.perf_tune_report.importers import ( + detect_variant_ab, + import_bundle_auto, + import_roofline_sweep_bundle, + import_variant_ab_bundle, +) from tools.perf_tune_report.importers.ncu_kernels import import_ncu_kernels from tools.perf_tune_report.importers.nsys_kernels import import_nsys_kernels from tools.perf_tune_report.kernel_profile import ( - KernelProfileResult, capture_kernel_profile, ) -from tools.perf_tune_report.graph_diff import ( - GraphDiffResult, - diff_graph_logs, -) from tools.perf_tune_report.kernel_reproducer import scaffold_reproducer from tools.perf_tune_report.orchestrator import CellPlan, run_campaign from tools.perf_tune_report.runners.aa_bench import run_cell as run_cell_aa +from tools.perf_tune_report.runners.aa_workload import redact_aiperf_command from tools.perf_tune_report.runners.aiperf_bench import run_cell as run_cell_aiperf from tools.perf_tune_report.runners.common import cell_config_from_dict from tools.perf_tune_report.runners.vllm_sweep import run_cell as run_cell_vllm_sweep from tools.perf_tune_report.schema import read_jsonl -from tools.perf_tune_report import champion_select as champion_select_lib -from tools.perf_tune_report.capture_signature import ( - build_plan as build_capture_plan, - materialize_reuse, -) from tools.perf_tune_report.value_view import ( build_value_view, default_registry_path, @@ -86,13 +91,13 @@ render_report, ) +logger = logging.getLogger(__name__) CONTRACT: dict[str, dict[str, Any]] = { "value_view": { "safety": "writes_artifacts", "required": (), - "optional": ("--registry", "--out", "--format", "--title", "--gpu-hr", - "--campaigns-dir", "--json"), + "optional": ("--registry", "--out", "--format", "--title", "--gpu-hr", "--campaigns-dir", "--json"), "json": True, "ack": None, "description": "Render the leadership value-prop ledger: join the curated " @@ -115,8 +120,7 @@ "campaign_init": { "safety": "writes_artifacts", "required": ("--config",), - "optional": ("--slug", "--experiment-id", "--family", "--evidence-bundle", - "--campaigns-dir", "--json"), + "optional": ("--slug", "--experiment-id", "--family", "--evidence-bundle", "--campaigns-dir", "--json"), "json": True, "ack": None, "description": "Scaffold a new campaign directory from a YAML config. Pass " @@ -126,9 +130,17 @@ "experiments_index": { "safety": "writes_artifacts", "required": (), - "optional": ("--family", "--out", "--include-s3", "--s3-endpoint", - "--s3-bucket", "--s3-access-key-file", "--s3-secret-key-file", - "--campaigns-dir", "--json"), + "optional": ( + "--family", + "--out", + "--include-s3", + "--s3-endpoint", + "--s3-bucket", + "--s3-access-key-file", + "--s3-secret-key-file", + "--campaigns-dir", + "--json", + ), "json": True, "ack": None, "description": "Enumerate all local campaigns into a cross-experiment index " @@ -138,9 +150,17 @@ "experiment_inventory": { "safety": "writes_artifacts", "required": (), - "optional": ("--bundle-root", "--out", "--include-s3", "--s3-endpoint", - "--s3-bucket", "--s3-access-key-file", "--s3-secret-key-file", - "--campaigns-dir", "--json"), + "optional": ( + "--bundle-root", + "--out", + "--include-s3", + "--s3-endpoint", + "--s3-bucket", + "--s3-access-key-file", + "--s3-secret-key-file", + "--campaigns-dir", + "--json", + ), "json": True, "ack": None, "description": "Canonical experiment count: unify local perf-report campaigns with " @@ -152,8 +172,15 @@ "import_model_eval": { "safety": "writes_artifacts", "required": ("--results", "--campaign", "--model", "--hardware", "--quant"), - "optional": ("--tensor-parallel", "--cell-id", "--parallel-strategy", - "--kv-cache-dtype", "--image", "--campaigns-dir", "--json"), + "optional": ( + "--tensor-parallel", + "--cell-id", + "--parallel-strategy", + "--kv-cache-dtype", + "--image", + "--campaigns-dir", + "--json", + ), "json": True, "ack": None, "description": "Import an lm-eval-harness results.json into a perf-report quality cell " @@ -163,10 +190,19 @@ "import_workloads": { "safety": "writes_artifacts", "required": ("--bench-dir", "--campaign", "--model", "--hardware", "--tensor-parallel"), - "optional": ("--quant", "--parallel-strategy", "--max-num-batched-tokens", - "--kv-cache-dtype", "--image", "--cudagraph-mode", - "--gpu-memory-utilization", "--bench-backend", "--dry-run", - "--campaigns-dir", "--json"), + "optional": ( + "--quant", + "--parallel-strategy", + "--max-num-batched-tokens", + "--kv-cache-dtype", + "--image", + "--cudagraph-mode", + "--gpu-memory-utilization", + "--bench-backend", + "--dry-run", + "--campaigns-dir", + "--json", + ), "json": True, "ack": None, "description": "Import a workload result directory (one -c.txt per " @@ -177,8 +213,16 @@ "trend_view": { "safety": "writes_artifacts", "required": (), - "optional": ("--metric", "--concurrency", "--regression-pct", "--hardware", - "--out", "--campaigns-dir", "--lake-dir", "--json"), + "optional": ( + "--metric", + "--concurrency", + "--regression-pct", + "--hardware", + "--out", + "--campaigns-dir", + "--lake-dir", + "--json", + ), "json": True, "ack": None, "description": "Longitudinal (model, variant_key) perf/quality trend across campaigns: " @@ -730,6 +774,7 @@ # Verb implementations # ---------------------------------------------------------------------------- + def cmd_campaign_init(args: argparse.Namespace) -> int: config_path = Path(args.config).expanduser().resolve() if not config_path.is_file(): @@ -767,6 +812,7 @@ def cmd_campaign_init(args: argparse.Namespace) -> int: # Lazy import: lake_writer's module top is light (pyarrow is imported # inside its functions), so this does not pull pyarrow into campaign_init. from tools.perf_tune_report.lake_writer import _CAMPAIGN_UTC_RE + if not _CAMPAIGN_UTC_RE.search(experiment_id): print( f"FATAL: --experiment-id {experiment_id!r} must contain a " @@ -822,9 +868,7 @@ def cmd_campaign_init(args: argparse.Namespace) -> int: if prov: import json as _json - (campaign_dir / "provenance.json").write_text( - _json.dumps(prov, indent=2, sort_keys=True) - ) + (campaign_dir / "provenance.json").write_text(_json.dumps(prov, indent=2, sort_keys=True)) if "provenance" not in config: # Append (not re-dump) so the original config bytes are preserved. import yaml as _yaml @@ -845,13 +889,10 @@ def cmd_campaign_init(args: argparse.Namespace) -> int: f"- backend(s): {sorted({c.get('backend', 'unspecified') for c in cells})}\n" f"- experiment_id: {experiment_id}\n" f"- family: {family}\n" - f"- evidence_bundle_path: {evidence_bundle}\n" - + prov_bullets + f"- evidence_bundle_path: {evidence_bundle}\n" + prov_bullets ) (campaign_dir / "SOURCE.md").write_text(source_md) - (campaign_dir / "summary.md").write_text( - "# Verdict\n\n_TBD: fill in after the run is complete._\n" - ) + (campaign_dir / "summary.md").write_text("# Verdict\n\n_TBD: fill in after the run is complete._\n") payload = { "tool": "perf_tune_report_campaign_init", @@ -889,8 +930,7 @@ def cmd_cell_run(args: argparse.Namespace) -> int: cells_by_id = {c["cell_id"]: c for c in config.get("cells", [])} if args.cell not in cells_by_id: print( - f"FATAL: cell {args.cell!r} not in campaign config; available: " - f"{sorted(cells_by_id)}", + f"FATAL: cell {args.cell!r} not in campaign config; available: {sorted(cells_by_id)}", file=sys.stderr, ) return 2 @@ -1057,14 +1097,13 @@ def cmd_cell_run(args: argparse.Namespace) -> int: "status": result.status, "row_count": result.row_count, "dry_run": result.dry_run, - "commands": result.commands, + "commands": [redact_aiperf_command(command) for command in result.commands], } if result.dataset_info: payload["dataset_info"] = result.dataset_info if result.spec_windows: payload["spec_decode"] = { - str(c): {"al": w["al"], "accept_rate": w["accept_rate"]} - for c, w in sorted(result.spec_windows.items()) + str(c): {"al": w["al"], "accept_rate": w["accept_rate"]} for c, w in sorted(result.spec_windows.items()) } else: print( @@ -1079,11 +1118,7 @@ def cmd_cell_run(args: argparse.Namespace) -> int: def cmd_value_view(args: argparse.Namespace) -> int: campaigns_root = resolve_campaigns_dir(args.campaigns_dir) - registry_path = ( - Path(args.registry).expanduser() - if args.registry - else default_registry_path(campaigns_root) - ) + registry_path = Path(args.registry).expanduser() if args.registry else default_registry_path(campaigns_root) if not registry_path.is_file(): print( f"FATAL: value-findings registry not found at {registry_path}; pass " @@ -1096,11 +1131,10 @@ def cmd_value_view(args: argparse.Namespace) -> int: # Resolve the $/GPU-hour: --gpu-hr override wins, else perf-tune-report/configs/cost.yaml # (the fleet is GB300), else GPU_HR_DEFAULT -- shared with fleet_leaderboard's knob. from tools.perf_tune_report.fleet_leaderboard import resolve_gpu_hr - gpu_hr = resolve_gpu_hr("GB300", campaigns_root.parent / "configs", - getattr(args, "gpu_hr", None)) + + gpu_hr = resolve_gpu_hr("GB300", campaigns_root.parent / "configs", getattr(args, "gpu_hr", None)) if getattr(args, "format", "table") == "report": - md = render_report(view, title=args.title or "Inference perf wins -- value prop", - gpu_hr=gpu_hr) + md = render_report(view, title=args.title or "Inference perf wins -- value prop", gpu_hr=gpu_hr) else: md = render_markdown(view, title=args.title or "Value ledger", gpu_hr=gpu_hr) out_path = Path(args.out).expanduser() if args.out else None @@ -1133,11 +1167,7 @@ def cmd_portability_view(args: argparse.Namespace) -> int: from tools.perf_tune_report.portability_view import build_portability, render_markdown campaigns_root = resolve_campaigns_dir(args.campaigns_dir) - registry_path = ( - Path(args.registry).expanduser() - if args.registry - else default_registry_path(campaigns_root) - ) + registry_path = Path(args.registry).expanduser() if args.registry else default_registry_path(campaigns_root) if not registry_path.is_file(): print( f"FATAL: value-findings registry not found at {registry_path}; pass " @@ -1174,13 +1204,12 @@ def cmd_champion_select(args: argparse.Namespace) -> int: campaigns_root = resolve_campaigns_dir(args.campaigns_dir) campaign_dir = resolve_campaign_dir(args.campaign, campaigns_root) workloads_present = ( - tuple(w.strip() for w in args.workloads_present.split(",") if w.strip()) - if args.workloads_present - else None + tuple(w.strip() for w in args.workloads_present.split(",") if w.strip()) if args.workloads_present else None + ) + require_workloads = ( + tuple(w.strip() for w in (args.require_workloads or "").split(",") if w.strip()) + or champion_select_lib.CANONICAL_WORKLOADS ) - require_workloads = tuple( - w.strip() for w in (args.require_workloads or "").split(",") if w.strip() - ) or champion_select_lib.CANONICAL_WORKLOADS try: result = champion_select_lib.select( campaign_dir, @@ -1207,9 +1236,7 @@ def cmd_champion_select(args: argparse.Namespace) -> int: json_path = campaign_dir / "champion_select.json" md_path = out_md or (campaign_dir / "CHAMPION.md") else: - json_path, md_path = champion_select_lib.write_outputs( - result, campaign_dir, out_md=out_md, title=args.title - ) + json_path, md_path = champion_select_lib.write_outputs(result, campaign_dir, out_md=out_md, title=args.title) if not args.json: print(champion_select_lib.render_markdown(result, title=args.title)) payload = { @@ -1239,10 +1266,7 @@ def cmd_champion_select(args: argparse.Namespace) -> int: def cmd_capture_plan(args: argparse.Namespace) -> int: campaigns_root = resolve_campaigns_dir(args.campaigns_dir) target_campaign = resolve_campaign_dir(args.campaign, campaigns_root) - source_campaigns = [ - resolve_campaign_dir(c, campaigns_root) - for c in (args.source_campaign or []) - ] + source_campaigns = [resolve_campaign_dir(c, campaigns_root) for c in (args.source_campaign or [])] if not source_campaigns: source_campaigns = [target_campaign] try: @@ -1251,16 +1275,18 @@ def cmd_capture_plan(args: argparse.Namespace) -> int: print(f"FATAL: {exc}", file=sys.stderr) return 2 payload = plan.to_dict() - payload.update({ - "tool": "perf_tune_report_capture_plan", - "library": "perf_tune_report", - "verb": "capture_plan", - "safety": CONTRACT["capture_plan"]["safety"], - "campaign_dir": str(target_campaign), - "cell_count": len(plan.cells), - "reuse_candidate_count": len(plan.reuse_candidates), - "missing_group_count": len(plan.missing_groups), - }) + payload.update( + { + "tool": "perf_tune_report_capture_plan", + "library": "perf_tune_report", + "verb": "capture_plan", + "safety": CONTRACT["capture_plan"]["safety"], + "campaign_dir": str(target_campaign), + "cell_count": len(plan.cells), + "reuse_candidate_count": len(plan.reuse_candidates), + "missing_group_count": len(plan.missing_groups), + } + ) if args.out: out = Path(args.out).expanduser() out.parent.mkdir(parents=True, exist_ok=True) @@ -1278,16 +1304,18 @@ def cmd_materialize_capture_reuse(args: argparse.Namespace) -> int: print(f"FATAL: {exc}", file=sys.stderr) return 2 payload = result.to_dict() - payload.update({ - "tool": "perf_tune_report_materialize_capture_reuse", - "library": "perf_tune_report", - "verb": "materialize_capture_reuse", - "safety": CONTRACT["materialize_capture_reuse"]["safety"], - "plan": str(plan_path), - "copied_count": len(result.copied), - "skipped_count": len(result.skipped), - "dry_run": args.dry_run, - }) + payload.update( + { + "tool": "perf_tune_report_materialize_capture_reuse", + "library": "perf_tune_report", + "verb": "materialize_capture_reuse", + "safety": CONTRACT["materialize_capture_reuse"]["safety"], + "plan": str(plan_path), + "copied_count": len(result.copied), + "skipped_count": len(result.skipped), + "dry_run": args.dry_run, + } + ) emit(payload, as_json=args.json) return 0 @@ -1401,8 +1429,7 @@ def cmd_report_render(args: argparse.Namespace) -> int: # + report_status.json. for omission in status.omitted_pages: print( - f"WARNING: omitted {omission['page']} -- {omission['why']} " - f"How to fix: {omission['how_to_fix']}", + f"WARNING: omitted {omission['page']} -- {omission['why']} How to fix: {omission['how_to_fix']}", file=sys.stderr, ) @@ -1555,10 +1582,7 @@ def _ordered_unique_hw(summary: Any) -> list[str]: def cmd_publish_to_lake(args: argparse.Namespace) -> int: """Publish a campaign's atlas + provenance as Parquet to S3.""" - if ( - not args.dry_run - and getattr(args, "i_understand_this_publishes_externally", False) is not True - ): + if not args.dry_run and getattr(args, "i_understand_this_publishes_externally", False) is not True: print( "FATAL: publish_to_lake writes to external S3 storage. Pass " "--i-understand-this-publishes-externally in the current command.", @@ -1683,22 +1707,34 @@ def cmd_publish_to_lake(args: argparse.Namespace) -> int: "row_count": result.quality.row_count, "skipped": result.quality.skipped, }, - **({"champion_v1": { - "local_path": str(result.champion.local_path), - "s3_key": result.champion.s3_key, - "size_bytes": result.champion.size_bytes, - "sha256": result.champion.sha256, - "row_count": result.champion.row_count, - "skipped": result.champion.skipped, - }} if result.champion is not None else {}), - **({"roofline_v1": { - "local_path": str(result.roofline.local_path), - "s3_key": result.roofline.s3_key, - "size_bytes": result.roofline.size_bytes, - "sha256": result.roofline.sha256, - "row_count": result.roofline.row_count, - "skipped": result.roofline.skipped, - }} if result.roofline is not None else {}), + **( + { + "champion_v1": { + "local_path": str(result.champion.local_path), + "s3_key": result.champion.s3_key, + "size_bytes": result.champion.size_bytes, + "sha256": result.champion.sha256, + "row_count": result.champion.row_count, + "skipped": result.champion.skipped, + } + } + if result.champion is not None + else {} + ), + **( + { + "roofline_v1": { + "local_path": str(result.roofline.local_path), + "s3_key": result.roofline.s3_key, + "size_bytes": result.roofline.size_bytes, + "sha256": result.roofline.sha256, + "row_count": result.roofline.row_count, + "skipped": result.roofline.skipped, + } + } + if result.roofline is not None + else {} + ), }, } @@ -1708,9 +1744,7 @@ def cmd_publish_to_lake(args: argparse.Namespace) -> int: # a real publish; the bundle path comes from the campaign SOURCE.md's # evidence_bundle_path (written by campaign_init --evidence-bundle). if not result.dry_run: - bundle_note = _append_lake_provenance_to_bundle( - campaign_dir, parse_source_md, result - ) + bundle_note = _append_lake_provenance_to_bundle(campaign_dir, parse_source_md, result) if bundle_note: payload["evidence_bundle_updated"] = bundle_note @@ -1765,6 +1799,7 @@ def cmd_experiments_index(args: argparse.Namespace) -> int: if getattr(args, "include_s3", False): from tools.perf_tune_report.experiments_index import enumerate_published_campaign_ids from tools.perf_tune_report.lake_writer import resolve_s3_config + try: cfg = resolve_s3_config( endpoint=args.s3_endpoint, @@ -1773,11 +1808,9 @@ def cmd_experiments_index(args: argparse.Namespace) -> int: secret_key_file=args.s3_secret_key_file, ) published = enumerate_published_campaign_ids(cfg, bucket=cfg.bucket) - print(f"== --include-s3: {len(published)} published campaign(s) in the lake ==", - file=sys.stderr) + print(f"== --include-s3: {len(published)} published campaign(s) in the lake ==", file=sys.stderr) except SystemExit as exc: - print(f"WARNING: --include-s3 skipped (S3 creds unavailable): {exc}", - file=sys.stderr) + print(f"WARNING: --include-s3 skipped (S3 creds unavailable): {exc}", file=sys.stderr) published = None except Exception as exc: # noqa: BLE001 - best-effort; never fail the index print(f"WARNING: --include-s3 lake enumeration failed: {exc}", file=sys.stderr) @@ -1823,6 +1856,7 @@ def cmd_experiment_inventory(args: argparse.Namespace) -> int: if getattr(args, "include_s3", False): from tools.perf_tune_report.experiments_index import enumerate_published_campaign_ids from tools.perf_tune_report.lake_writer import resolve_s3_config + try: cfg = resolve_s3_config( endpoint=args.s3_endpoint, @@ -1831,11 +1865,9 @@ def cmd_experiment_inventory(args: argparse.Namespace) -> int: secret_key_file=args.s3_secret_key_file, ) published = enumerate_published_campaign_ids(cfg, bucket=cfg.bucket) - print(f"== --include-s3: {len(published)} published campaign(s) in the lake ==", - file=sys.stderr) + print(f"== --include-s3: {len(published)} published campaign(s) in the lake ==", file=sys.stderr) except SystemExit as exc: - print(f"WARNING: --include-s3 skipped (S3 creds unavailable): {exc}", - file=sys.stderr) + print(f"WARNING: --include-s3 skipped (S3 creds unavailable): {exc}", file=sys.stderr) published = None except Exception as exc: # noqa: BLE001 - best-effort; never fail the inventory print(f"WARNING: --include-s3 lake enumeration failed: {exc}", file=sys.stderr) @@ -1886,8 +1918,7 @@ def cmd_trend_view(args: argparse.Namespace) -> int: print(f"FATAL: campaigns dir does not exist: {campaigns_root}", file=sys.stderr) return 2 rows = read_all_rows(campaigns_root, hardware_filter=args.hardware) - view = build_trends(rows, metric=args.metric, concurrency=args.concurrency, - regression_pct=args.regression_pct) + view = build_trends(rows, metric=args.metric, concurrency=args.concurrency, regression_pct=args.regression_pct) md = render_markdown(view) out_path = Path(args.out).expanduser() if args.out else None if out_path is not None: @@ -1924,11 +1955,16 @@ def cmd_import_model_eval(args: argparse.Namespace) -> int: return 2 try: result = import_model_eval( - results, campaign_dir, - model=args.model, hardware=args.hardware, quant=args.quant, - tensor_parallel=args.tensor_parallel, cell_id=args.cell_id, + results, + campaign_dir, + model=args.model, + hardware=args.hardware, + quant=args.quant, + tensor_parallel=args.tensor_parallel, + cell_id=args.cell_id, parallel_strategy=args.parallel_strategy, - kv_cache_dtype=args.kv_cache_dtype, image=args.image, + kv_cache_dtype=args.kv_cache_dtype, + image=args.image, ) except ValueError as exc: print(f"FATAL: {exc}", file=sys.stderr) @@ -1969,17 +2005,23 @@ def cmd_import_workloads(args: argparse.Namespace) -> int: args.gpu_memory_utilization = _dd["gpu_memory_utilization"] if _dd.get("cudagraph_mode") == "eager": args.cudagraph_mode = "eager" + kv_cache_dtype = str(args.kv_cache_dtype or "unknown") try: result = import_workloads( - bench_dir, campaign_dir, - model=args.model, hardware=args.hardware, - tensor_parallel=args.tensor_parallel, quant=args.quant, + bench_dir, + campaign_dir, + model=args.model, + hardware=args.hardware, + tensor_parallel=args.tensor_parallel, + quant=args.quant, parallel_strategy=args.parallel_strategy, max_num_batched_tokens=args.max_num_batched_tokens, - kv_cache_dtype=args.kv_cache_dtype, image=args.image, + kv_cache_dtype=kv_cache_dtype, + image=args.image, cudagraph_mode=args.cudagraph_mode, gpu_memory_utilization=args.gpu_memory_utilization, - bench_backend=args.bench_backend, dry_run=args.dry_run, + bench_backend=args.bench_backend, + dry_run=args.dry_run, ).to_dict() except (FileNotFoundError, ValueError) as exc: print(f"FATAL: {exc}", file=sys.stderr) @@ -2012,6 +2054,7 @@ def cmd_fleet_leaderboard(args: argparse.Namespace) -> int: # Resolve the $/GPU-hour: --gpu-hr override wins, else perf-tune-report/configs/cost.yaml. from tools.perf_tune_report.fleet_leaderboard import resolve_gpu_hr + gpu_hr = resolve_gpu_hr(hw, campaigns_root.parent / "configs", args.gpu_hr) # Default output: the perf-report bundle (campaigns_root's parent), alongside configs. @@ -2081,16 +2124,18 @@ def cmd_campaign_run(args: argparse.Namespace) -> int: helm_overrides = c.get("helm_overrides", {}) or {} if not isinstance(helm_overrides, dict): raise TypeError(f"cell {cell_id}: helm_overrides must be a mapping") - cells.append(CellPlan( - id=cell_id, - backend=backend, - concurrencies=tuple(int(value) for value in c.get("concurrencies", [])), - helm_overrides=helm_overrides, - profile=c.get("profile", {}) or {}, - notes=c.get("notes", ""), - runner_config=dict(c), - backend_config=backend_config, - )) + cells.append( + CellPlan( + id=cell_id, + backend=backend, + concurrencies=tuple(int(value) for value in c.get("concurrencies", [])), + helm_overrides=helm_overrides, + profile=c.get("profile", {}) or {}, + notes=c.get("notes", ""), + runner_config=dict(c), + backend_config=backend_config, + ) + ) except (KeyError, TypeError, ValueError) as exc: print(f"FATAL: cell config invalid: {exc}", file=sys.stderr) return 2 @@ -2105,26 +2150,36 @@ def cmd_campaign_run(args: argparse.Namespace) -> int: endpoint_only_backends = ("aiperf", "aa") for cell in cells: endpoint_only = cell.backend in endpoint_only_backends - plan.append({ - "cell_id": cell.id, - "backend": cell.backend, - "concurrencies": list(cell.concurrencies), - "steps": [ - "1. drain Slurm-on-K8s co-tenants" if campaign_cfg.get("drain_nodes") else "1. (no Slurm-on-K8s nodes; drain skipped)", - f"2. (backend={cell.backend} targets existing endpoint; helm_upgrade skipped)" if endpoint_only else "2. helm upgrade with cell helm_overrides", - f"3. (backend={cell.backend} targets existing endpoint; warmup skipped)" if endpoint_only else "3. warmup probe (5-prompt c=4)", - f"4. cell_run backend={cell.backend}", - "5. zymtrace anchored query" if cell.profile.get("zymtrace", "on") == "on" else "5. (zymtrace disabled)", - "6. import_perf_bench (raw -> normalized.json)", - "7. atlas_aggregate (campaign rollup)", - "7b. dcgm_correlate (byte-grounding -> dcgm_correlation.json, pages 6/6b) if cells//dcgm-frozen.yaml present", - "8. report_render (after-each-cell PDF refresh)", - "9. baseline_record", - "10. baseline_diff (returns verdict)", - "FINALLY: Slurm-on-K8s resume (always, even on Ctrl-C)", - ], - "notes": cell.notes, - }) + plan.append( + { + "cell_id": cell.id, + "backend": cell.backend, + "concurrencies": list(cell.concurrencies), + "steps": [ + "1. drain Slurm-on-K8s co-tenants" + if campaign_cfg.get("drain_nodes") + else "1. (no Slurm-on-K8s nodes; drain skipped)", + f"2. (backend={cell.backend} targets existing endpoint; helm_upgrade skipped)" + if endpoint_only + else "2. helm upgrade with cell helm_overrides", + f"3. (backend={cell.backend} targets existing endpoint; warmup skipped)" + if endpoint_only + else "3. warmup probe (5-prompt c=4)", + f"4. cell_run backend={cell.backend}", + "5. zymtrace anchored query" + if cell.profile.get("zymtrace", "on") == "on" + else "5. (zymtrace disabled)", + "6. import_perf_bench (raw -> normalized.json)", + "7. atlas_aggregate (campaign rollup)", + "7b. dcgm_correlate (byte-grounding -> dcgm_correlation.json, pages 6/6b) if cells//dcgm-frozen.yaml present", + "8. report_render (after-each-cell PDF refresh)", + "9. baseline_record", + "10. baseline_diff (returns verdict)", + "FINALLY: Slurm-on-K8s resume (always, even on Ctrl-C)", + ], + "notes": cell.notes, + } + ) payload = { "tool": "perf_tune_report_campaign_run", "library": "perf_tune_report", @@ -2152,14 +2207,11 @@ def cmd_campaign_run(args: argparse.Namespace) -> int: step_fns = production_step_fns() target_namespace = campaign_cfg.get("target_namespace", "inference") - manages_deployment = any( - cell.backend not in ("aiperf", "aa") for cell in cells - ) + manages_deployment = any(cell.backend not in ("aiperf", "aa") for cell in cells) target_release = campaign_cfg.get("target_release", "") if manages_deployment and not target_release: print( - "FATAL: campaign.target_release is required for non-dry-run " - "campaigns with deployment-managed cells.", + "FATAL: campaign.target_release is required for non-dry-run campaigns with deployment-managed cells.", file=sys.stderr, ) return 2 @@ -2167,16 +2219,11 @@ def cmd_campaign_run(args: argparse.Namespace) -> int: chart_dir_str = campaign_cfg.get("chart_dir", "") if manages_deployment and not chart_dir_str: print( - "FATAL: campaign.chart_dir is required for non-dry-run " - "campaigns with deployment-managed cells.", + "FATAL: campaign.chart_dir is required for non-dry-run campaigns with deployment-managed cells.", file=sys.stderr, ) return 2 - chart_dir = ( - Path(chart_dir_str).expanduser().resolve() - if chart_dir_str - else campaign_dir - ) + chart_dir = Path(chart_dir_str).expanduser().resolve() if chart_dir_str else campaign_dir base_values_str = campaign_cfg.get("base_values", "") base_values = Path(base_values_str).expanduser().resolve() if base_values_str else Path() @@ -2184,8 +2231,6 @@ def cmd_campaign_run(args: argparse.Namespace) -> int: drain_nodes = tuple(campaign_cfg.get("drain_nodes", []) or ()) comparator = campaign_cfg.get("comparator_baseline", "") or "" - from tools.perf_tune_report.orchestrator import run_campaign - try: result = run_campaign( cells, @@ -2356,7 +2401,8 @@ def _run_manifest_descriptor_defaults(bundle: Path) -> dict[str, Any]: continue try: m = json.loads(p.read_text()) - except Exception: + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + logger.debug("Skipping unreadable run manifest %s: %s", p, exc) continue if not isinstance(m, dict): continue @@ -2455,6 +2501,9 @@ def cmd_import_perf_bench(args: argparse.Namespace) -> int: # importer="inference_drive_load" attribute; the bench-serve one does # not because it uses the legacy result shape. importer_name = getattr(result, "importer", "inference_perf_bench") + cell_dir = getattr(result, "cell_dir", None) + normalized_path = getattr(result, "normalized_path", None) + imported_bundle_path = getattr(result, "bundle_path", None) payload = { "tool": "perf_tune_report_import_perf_bench", @@ -2466,13 +2515,9 @@ def cmd_import_perf_bench(args: argparse.Namespace) -> int: # lws_summary emits multiple cells -> LwsSummaryImportResult has no single # cell_id/cell_dir/normalized_path; guard so the multi-cell path doesn't crash. "cell_id": getattr(result, "cell_id", None), - "cell_dir": (str(result.cell_dir) if getattr(result, "cell_dir", None) else None), - "normalized_path": ( - str(result.normalized_path) - if getattr(result, "normalized_path", None) - else None - ), - "bundle_path": (str(result.bundle_path) if getattr(result, "bundle_path", None) else None), + "cell_dir": str(cell_dir) if cell_dir else None, + "normalized_path": str(normalized_path) if normalized_path else None, + "bundle_path": str(imported_bundle_path) if imported_bundle_path else None, "row_count": result.row_count, "concurrencies": getattr(result, "concurrencies", None), "k_values": getattr(result, "k_values", None), @@ -2499,11 +2544,26 @@ def cmd_import_variant_ab(args: argparse.Namespace) -> int: overrides: dict[str, Any] = {} for attr in ( - "model", "hardware", "quant", "tensor_parallel", "parallel_strategy", - "mtp", "max_num_batched_tokens", "cache_mode", "notes", - "dataset", "cudagraph_mode", "enforce_eager", "gpu_memory_utilization", - "kv_cache_dtype", "image", "delivery", "overlay_mode", "patch_files", - "data_parallel", "pipeline_parallel", + "model", + "hardware", + "quant", + "tensor_parallel", + "parallel_strategy", + "mtp", + "max_num_batched_tokens", + "cache_mode", + "notes", + "dataset", + "cudagraph_mode", + "enforce_eager", + "gpu_memory_utilization", + "kv_cache_dtype", + "image", + "delivery", + "overlay_mode", + "patch_files", + "data_parallel", + "pipeline_parallel", ): val = getattr(args, attr, None) if val is not None: @@ -2548,13 +2608,29 @@ def cmd_import_roofline_sweep(args: argparse.Namespace) -> int: overrides: dict[str, Any] = {} for attr in ( - "cell_id", "model", "hardware", "quant", "kv_dtype", "tensor_parallel", - "parallel_strategy", "mtp", "max_num_batched_tokens", "cache_mode", - "model_config_path", "kv_cache_dtype", + "cell_id", + "model", + "hardware", + "quant", + "kv_dtype", + "tensor_parallel", + "parallel_strategy", + "mtp", + "max_num_batched_tokens", + "cache_mode", + "model_config_path", + "kv_cache_dtype", # full-context descriptors (so roofline cells can pass publish --strict) - "dataset", "cudagraph_mode", "enforce_eager", "gpu_memory_utilization", - "image", "delivery", "overlay_mode", "patch_files", - "data_parallel", "pipeline_parallel", + "dataset", + "cudagraph_mode", + "enforce_eager", + "gpu_memory_utilization", + "image", + "delivery", + "overlay_mode", + "patch_files", + "data_parallel", + "pipeline_parallel", ): val = getattr(args, attr, None) if val is not None: @@ -2606,8 +2682,7 @@ def _resolve_ceilings_yaml(explicit: str | None, campaign_dir: Path) -> Path | N # Canonical bundle name first, then a name-agnostic bundle-root fallback # (/configs/sol-ceilings.yaml) so a future submodule rename needs no # code edit; the campaign's own bundle root is reached before higher ancestors. - relcands = (Path("perf-tune-report") / "configs" / "sol-ceilings.yaml", - Path("configs") / "sol-ceilings.yaml") + relcands = (Path("perf-tune-report") / "configs" / "sol-ceilings.yaml", Path("configs") / "sol-ceilings.yaml") cur = campaign_dir.resolve() for parent in [cur, *cur.parents]: for relpath in relcands: @@ -2681,9 +2756,7 @@ def cmd_dcgm_correlate(args: argparse.Namespace) -> int: return 0 try: - result = correlate_from_frozen( - frozen, ceilings, cell_dir=cell_dir, kernels_json_path=kernels_json_path - ) + result = correlate_from_frozen(frozen, ceilings, cell_dir=cell_dir, kernels_json_path=kernels_json_path) except FrozenYamlMalformed as exc: print(f"FATAL: {exc}", file=sys.stderr) return 2 @@ -2746,9 +2819,7 @@ def cmd_import_nsys(args: argparse.Namespace) -> int: "cell_id": args.cell_id, "cell_dir": str(cell_dir), "bundle_path": str(bundle_path), - "kernels_json_path": ( - str(result.kernels_json_path) if result.kernels_json_path else None - ), + "kernels_json_path": (str(result.kernels_json_path) if result.kernels_json_path else None), "top_kernel_count": result.top_kernel_count, "category_count": result.category_count, "skipped_reason": result.skipped_reason, @@ -2792,9 +2863,7 @@ def cmd_import_ncu(args: argparse.Namespace) -> int: "cell_dir": str(cell_dir), "bundle_path": str(bundle_path), "hw_key": args.hw_key, - "ncu_kernels_json_path": ( - str(result.ncu_kernels_json_path) if result.ncu_kernels_json_path else None - ), + "ncu_kernels_json_path": (str(result.ncu_kernels_json_path) if result.ncu_kernels_json_path else None), "kernel_count": result.kernel_count, "skipped_reason": result.skipped_reason, "dry_run": args.dry_run, @@ -2868,9 +2937,7 @@ def cmd_report_smoke(args: argparse.Namespace) -> int: out_pdf, title=args.title or "glm5p1 benchmark report", variants_line=( - "Variants: GLM-5.1-FP8 (FP8, H100) | " - "GLM-5.1-NVFP4 (NVFP4, B200) | " - "GLM-5.1-NVFP4 (NVFP4, GB300)" + "Variants: GLM-5.1-FP8 (FP8, H100) | GLM-5.1-NVFP4 (NVFP4, B200) | GLM-5.1-NVFP4 (NVFP4, GB300)" ), data_source_line=( "Data source: synthetic chat | each concurrency point targeted 600s " @@ -2906,6 +2973,7 @@ def cmd_report_smoke(args: argparse.Namespace) -> int: # Parser plumbing # ---------------------------------------------------------------------------- + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="perftunereport", @@ -2937,57 +3005,56 @@ def build_parser() -> argparse.ArgumentParser: ci.add_argument( "--evidence-bundle", default=None, - help="Path to the evidence bundle this campaign is derived from. Falls back " - "to config evidence_bundle_path:.", + help="Path to the evidence bundle this campaign is derived from. Falls back to config evidence_bundle_path:.", ) ci.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") ci.add_argument("--json", action="store_true", help="Emit JSON envelope") ci.set_defaults(func=cmd_campaign_init) # experiments_index - ei = sub.add_parser("experiments_index", - description=CONTRACT["experiments_index"]["description"]) - ei.add_argument("--family", default=None, - help="Only include experiments in this family (e.g. nvfp4-kv)") - ei.add_argument("--out", default=None, - help="Output dir (default: the perf-report bundle, campaigns' parent)") - ei.add_argument("--include-s3", action="store_true", - help="Also enumerate the lake's campaign_v1 prefix to mark each row's " - "published_to_lake (best-effort; warns + continues if S3 creds absent)") + ei = sub.add_parser("experiments_index", description=CONTRACT["experiments_index"]["description"]) + ei.add_argument("--family", default=None, help="Only include experiments in this family (e.g. nvfp4-kv)") + ei.add_argument("--out", default=None, help="Output dir (default: the perf-report bundle, campaigns' parent)") + ei.add_argument( + "--include-s3", + action="store_true", + help="Also enumerate the lake's campaign_v1 prefix to mark each row's " + "published_to_lake (best-effort; warns + continues if S3 creds absent)", + ) ei.add_argument("--s3-endpoint", default=None, help="S3 endpoint (with --include-s3)") ei.add_argument("--s3-bucket", default=None, help="S3 bucket (with --include-s3)") - ei.add_argument("--s3-access-key-file", default=None, - help="File with S3 access key (with --include-s3)") - ei.add_argument("--s3-secret-key-file", default=None, - help="File with S3 secret key (with --include-s3)") + ei.add_argument("--s3-access-key-file", default=None, help="File with S3 access key (with --include-s3)") + ei.add_argument("--s3-secret-key-file", default=None, help="File with S3 secret key (with --include-s3)") ei.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") ei.add_argument("--json", action="store_true", help="Emit JSON envelope") ei.set_defaults(func=cmd_experiments_index) # experiment_inventory - einv = sub.add_parser("experiment_inventory", - description=CONTRACT["experiment_inventory"]["description"]) - einv.add_argument("--bundle-root", action="append", default=None, - help="Evidence-bundle tree root to also walk for run-id-stamped bundles " - "(repeatable; e.g. a *-deploy dir). Campaigns are always counted.") - einv.add_argument("--out", default=None, - help="Output dir (default: the perf-report bundle, campaigns' parent)") - einv.add_argument("--include-s3", action="store_true", - help="Also enumerate the lake's campaign_v1 prefix for the published count " - "(best-effort; warns + continues if S3 creds absent)") + einv = sub.add_parser("experiment_inventory", description=CONTRACT["experiment_inventory"]["description"]) + einv.add_argument( + "--bundle-root", + action="append", + default=None, + help="Evidence-bundle tree root to also walk for run-id-stamped bundles " + "(repeatable; e.g. a *-deploy dir). Campaigns are always counted.", + ) + einv.add_argument("--out", default=None, help="Output dir (default: the perf-report bundle, campaigns' parent)") + einv.add_argument( + "--include-s3", + action="store_true", + help="Also enumerate the lake's campaign_v1 prefix for the published count " + "(best-effort; warns + continues if S3 creds absent)", + ) einv.add_argument("--s3-endpoint", default=None, help="S3 endpoint (with --include-s3)") einv.add_argument("--s3-bucket", default=None, help="S3 bucket (with --include-s3)") - einv.add_argument("--s3-access-key-file", default=None, - help="File with S3 access key (with --include-s3)") - einv.add_argument("--s3-secret-key-file", default=None, - help="File with S3 secret key (with --include-s3)") + einv.add_argument("--s3-access-key-file", default=None, help="File with S3 access key (with --include-s3)") + einv.add_argument("--s3-secret-key-file", default=None, help="File with S3 secret key (with --include-s3)") einv.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") einv.add_argument("--json", action="store_true", help="Emit JSON envelope") einv.set_defaults(func=cmd_experiment_inventory) # import_model_eval - ime = sub.add_parser("import_model_eval", - description=CONTRACT["import_model_eval"]["description"]) + ime = sub.add_parser("import_model_eval", description=CONTRACT["import_model_eval"]["description"]) ime.add_argument("--results", required=True, help="lm-eval-harness results.json path") ime.add_argument("--campaign", required=True, help="Campaign dir path OR slug") ime.add_argument("--model", required=True, help="Model (matches the serving campaign's model)") @@ -2995,8 +3062,7 @@ def build_parser() -> argparse.ArgumentParser: ime.add_argument("--quant", required=True, help="Quant (NVFP4 / FP8 / ...)") ime.add_argument("--tensor-parallel", type=int, default=1, dest="tensor_parallel") ime.add_argument("--cell-id", default="model-eval", dest="cell_id") - ime.add_argument("--parallel-strategy", default="TP", dest="parallel_strategy", - choices=("TP", "EP")) + ime.add_argument("--parallel-strategy", default="TP", dest="parallel_strategy", choices=("TP", "EP")) ime.add_argument("--kv-cache-dtype", default="unknown", dest="kv_cache_dtype") ime.add_argument("--image", default="unknown", help="Serving image tag the eval ran against") ime.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") @@ -3004,28 +3070,37 @@ def build_parser() -> argparse.ArgumentParser: ime.set_defaults(func=cmd_import_model_eval) # import_workloads - iw = sub.add_parser("import_workloads", - description=CONTRACT["import_workloads"]["description"]) - iw.add_argument("--bench-dir", required=True, dest="bench_dir", - help="Workload result dir (-c.txt + bench-workloads.json)") + iw = sub.add_parser("import_workloads", description=CONTRACT["import_workloads"]["description"]) + iw.add_argument( + "--bench-dir", + required=True, + dest="bench_dir", + help="Workload result dir (-c.txt + bench-workloads.json)", + ) iw.add_argument("--campaign", required=True, help="Campaign dir path OR slug") iw.add_argument("--model", required=True, help="Served model (matches the campaign's model)") iw.add_argument("--hardware", required=True, help="Hardware token (GB300 / B200)") iw.add_argument("--tensor-parallel", required=True, type=int, dest="tensor_parallel") iw.add_argument("--quant", default="NVFP4", help="Quant (NVFP4 / FP8 / ...)") - iw.add_argument("--parallel-strategy", default="TP", dest="parallel_strategy", - choices=("TP", "EP")) - iw.add_argument("--max-num-batched-tokens", type=int, default=0, - dest="max_num_batched_tokens") - iw.add_argument("--kv-cache-dtype", default="unknown", dest="kv_cache_dtype", - help="Serve kv-cache dtype (required-context: set for a publish --strict run)") - iw.add_argument("--image", default="unknown", - help="Serving image tag (required-context: set for a publish --strict run)") + iw.add_argument("--parallel-strategy", default="TP", dest="parallel_strategy", choices=("TP", "EP")) + iw.add_argument("--max-num-batched-tokens", type=int, default=0, dest="max_num_batched_tokens") + iw.add_argument( + "--kv-cache-dtype", + default="unknown", + dest="kv_cache_dtype", + help="Serve kv-cache dtype (required-context: set for a publish --strict run)", + ) + iw.add_argument( + "--image", default="unknown", help="Serving image tag (required-context: set for a publish --strict run)" + ) iw.add_argument("--cudagraph-mode", default="full", dest="cudagraph_mode") - iw.add_argument("--gpu-memory-utilization", type=float, default=None, - dest="gpu_memory_utilization") - iw.add_argument("--bench-backend", default="openai", dest="bench_backend", - help="Benchmark client backend recorded on imported rows") + iw.add_argument("--gpu-memory-utilization", type=float, default=None, dest="gpu_memory_utilization") + iw.add_argument( + "--bench-backend", + default="openai", + dest="bench_backend", + help="Benchmark client backend recorded on imported rows", + ) iw.add_argument("--dry-run", action="store_true", help="Parse + report, write nothing") iw.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") iw.add_argument("--json", action="store_true", help="Emit JSON envelope") @@ -3033,32 +3108,47 @@ def build_parser() -> argparse.ArgumentParser: # trend_view tv = sub.add_parser("trend_view", description=CONTRACT["trend_view"]["description"]) - tv.add_argument("--metric", default="output_tps_per_gpu", - help="atlas metric to trend (output_tps_per_gpu | tpot_median_ms | ttft_avg_ms | ...)") - tv.add_argument("--concurrency", type=int, default=None, - help="filter to one concurrency (else each c is its own trend line)") - tv.add_argument("--regression-pct", type=float, default=10.0, dest="regression_pct", - help="abs %% move in the wrong direction that flags a regression (default 10)") + tv.add_argument( + "--metric", + default="output_tps_per_gpu", + help="atlas metric to trend (output_tps_per_gpu | tpot_median_ms | ttft_avg_ms | ...)", + ) + tv.add_argument( + "--concurrency", type=int, default=None, help="filter to one concurrency (else each c is its own trend line)" + ) + tv.add_argument( + "--regression-pct", + type=float, + default=10.0, + dest="regression_pct", + help="abs %% move in the wrong direction that flags a regression (default 10)", + ) tv.add_argument("--hardware", default="GB300", help="hardware token filter (default GB300)") tv.add_argument("--out", default=None, help="write the trend markdown to this path") tv.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") - tv.add_argument("--lake-dir", default=None, - help="Read published atlas_v1 parquet from a pulled lake snapshot " - "(joins campaign_v1 vllm_commit as the engine-version axis); " - "local campaigns are the default") + tv.add_argument( + "--lake-dir", + default=None, + help="Read published atlas_v1 parquet from a pulled lake snapshot " + "(joins campaign_v1 vllm_commit as the engine-version axis); " + "local campaigns are the default", + ) tv.add_argument("--json", action="store_true", help="Emit JSON envelope") tv.set_defaults(func=cmd_trend_view) # fleet_leaderboard - fl = sub.add_parser("fleet_leaderboard", - description=CONTRACT["fleet_leaderboard"]["description"]) - fl.add_argument("--hardware", default="GB300", - help="Hardware token to filter atlas rows + name outputs (default: GB300)") - fl.add_argument("--gpu-hr", type=float, default=None, - help="$/GPU-hour for the cost columns (overrides perf-tune-report/configs/cost.yaml; " - "default: resolve from cost.yaml by hardware, else 8.60)") - fl.add_argument("--out", default=None, - help="Output dir (default: the perf-report bundle, campaigns' parent)") + fl = sub.add_parser("fleet_leaderboard", description=CONTRACT["fleet_leaderboard"]["description"]) + fl.add_argument( + "--hardware", default="GB300", help="Hardware token to filter atlas rows + name outputs (default: GB300)" + ) + fl.add_argument( + "--gpu-hr", + type=float, + default=None, + help="$/GPU-hour for the cost columns (overrides perf-tune-report/configs/cost.yaml; " + "default: resolve from cost.yaml by hardware, else 8.60)", + ) + fl.add_argument("--out", default=None, help="Output dir (default: the perf-report bundle, campaigns' parent)") fl.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") fl.add_argument("--json", action="store_true", help="Emit JSON envelope") fl.set_defaults(func=cmd_fleet_leaderboard) @@ -3189,7 +3279,12 @@ def build_parser() -> argparse.ArgumentParser: cp = sub.add_parser("campaign_run", description=CONTRACT["campaign_run"]["description"]) cp.add_argument("--config", required=True, help="Path to a campaign matrix YAML") cp.add_argument("--campaign", required=True, help="Campaign dir path OR slug") - cp.add_argument("--continue-on-red", action="store_true", dest="continue_on_red", help="Continue past RED cell verdicts instead of fail-fast") + cp.add_argument( + "--continue-on-red", + action="store_true", + dest="continue_on_red", + help="Continue past RED cell verdicts instead of fail-fast", + ) cp.add_argument("--dry-run", action="store_true", help="Print the 10-step plan JSON without submitting jobs") cp.add_argument( "--i-understand-this-mutates-cluster", @@ -3256,16 +3351,64 @@ def build_parser() -> argparse.ArgumentParser: # Full-context descriptor overrides follow AGENTS.md "Benchmark methodology hygiene". # They are required via a flag or bundle metadata for a measured campaign # to pass publish/render --strict. - ip.add_argument("--dataset", dest="dataset", default=None, help="Workload dataset (random|sharegpt|sonnet|aa|code|...) -- full-context descriptor") - ip.add_argument("--cudagraph-mode", dest="cudagraph_mode", default=None, help="full|piecewise|none|eager -- full-context descriptor (the eager/cudagraph trap)") - ip.add_argument("--enforce-eager", dest="enforce_eager", type=lambda s: s.lower() in ("true", "1", "yes"), default=None, help="true|false -- sets cudagraph_mode=eager when no explicit --cudagraph-mode") - ip.add_argument("--gpu-memory-utilization", dest="gpu_memory_utilization", type=float, default=None, help="gmu the number was measured at -- full-context descriptor") - ip.add_argument("--kv-cache-dtype", dest="kv_cache_dtype", default=None, help="fp8_e4m3|bf16|nvfp4|... -- full-context descriptor") - ip.add_argument("--image", dest="image", default=None, help="serving image tag / vllm commit -- full-context descriptor") - ip.add_argument("--delivery", dest="delivery", default=None, choices=("image", "overlay", "patchedVllm", "infr-patch"), help="how code reached the cluster (delivery ladder) -- full-context descriptor") - ip.add_argument("--overlay-mode", dest="overlay_mode", default=None, choices=("subpath", "patchset-initcontainer", "pythonpath-sitecustomize"), help="overlay sub-tier when --delivery=overlay") - ip.add_argument("--patch-files", dest="patch_files", default=None, help="comma-separated patch files applied (e.g. infr 0006..0026) -- full-context descriptor") - ip.add_argument("--data-parallel", dest="data_parallel", type=int, default=None, help="DP replica count (default 1)") + ip.add_argument( + "--dataset", + dest="dataset", + default=None, + help="Workload dataset (random|sharegpt|sonnet|aa|code|...) -- full-context descriptor", + ) + ip.add_argument( + "--cudagraph-mode", + dest="cudagraph_mode", + default=None, + help="full|piecewise|none|eager -- full-context descriptor (the eager/cudagraph trap)", + ) + ip.add_argument( + "--enforce-eager", + dest="enforce_eager", + type=lambda s: s.lower() in ("true", "1", "yes"), + default=None, + help="true|false -- sets cudagraph_mode=eager when no explicit --cudagraph-mode", + ) + ip.add_argument( + "--gpu-memory-utilization", + dest="gpu_memory_utilization", + type=float, + default=None, + help="gmu the number was measured at -- full-context descriptor", + ) + ip.add_argument( + "--kv-cache-dtype", + dest="kv_cache_dtype", + default=None, + help="fp8_e4m3|bf16|nvfp4|... -- full-context descriptor", + ) + ip.add_argument( + "--image", dest="image", default=None, help="serving image tag / vllm commit -- full-context descriptor" + ) + ip.add_argument( + "--delivery", + dest="delivery", + default=None, + choices=("image", "overlay", "patchedVllm", "infr-patch"), + help="how code reached the cluster (delivery ladder) -- full-context descriptor", + ) + ip.add_argument( + "--overlay-mode", + dest="overlay_mode", + default=None, + choices=("subpath", "patchset-initcontainer", "pythonpath-sitecustomize"), + help="overlay sub-tier when --delivery=overlay", + ) + ip.add_argument( + "--patch-files", + dest="patch_files", + default=None, + help="comma-separated patch files applied (e.g. infr 0006..0026) -- full-context descriptor", + ) + ip.add_argument( + "--data-parallel", dest="data_parallel", type=int, default=None, help="DP replica count (default 1)" + ) ip.add_argument("--pipeline-parallel", dest="pipeline_parallel", type=int, default=None, help="PP size (default 1)") ip.add_argument( "--concurrency", @@ -3314,25 +3457,90 @@ def build_parser() -> argparse.ArgumentParser: irs.add_argument("--model", default=None, help="Model name (default: from manifest / zai-org/GLM-5.1)") irs.add_argument("--hardware", default=None, help="e.g. B200, GB300, H100 (default: GB300)") irs.add_argument("--quant", default=None, help="NVFP4 | FP8 | BF16 (default: NVFP4)") - irs.add_argument("--kv-dtype", default=None, dest="kv_dtype", help="KV cache dtype for byte-grounded HBM utilization (default: fp8)") - irs.add_argument("--kv-cache-dtype", default=None, dest="kv_cache_dtype", help="Alias for --kv-dtype; matches atlas full-context naming") - irs.add_argument("--model-config", default=None, dest="model_config_path", help="Path to the model config.json (for analytical roofline math when the family is not in the registry)") + irs.add_argument( + "--kv-dtype", + default=None, + dest="kv_dtype", + help="KV cache dtype for byte-grounded HBM utilization (default: fp8)", + ) + irs.add_argument( + "--kv-cache-dtype", + default=None, + dest="kv_cache_dtype", + help="Alias for --kv-dtype; matches atlas full-context naming", + ) + irs.add_argument( + "--model-config", + default=None, + dest="model_config_path", + help="Path to the model config.json (for analytical roofline math when the family is not in the registry)", + ) irs.add_argument("--tensor-parallel", type=int, default=None, dest="tensor_parallel", help="TP size (default: 4)") - irs.add_argument("--parallel-strategy", default=None, dest="parallel_strategy", choices=("TP", "EP"), help="TP or EP (default: TP)") - irs.add_argument("--mtp", type=lambda s: s.lower() in ("true", "1", "yes"), default=None, help="true|false (default: false)") - irs.add_argument("--max-num-batched-tokens", type=int, default=None, dest="max_num_batched_tokens", help="default 12288") - irs.add_argument("--cache-mode", dest="cache_mode", default=None, choices=("warm", "cold", "unknown"), help="methodology label") + irs.add_argument( + "--parallel-strategy", + default=None, + dest="parallel_strategy", + choices=("TP", "EP"), + help="TP or EP (default: TP)", + ) + irs.add_argument( + "--mtp", type=lambda s: s.lower() in ("true", "1", "yes"), default=None, help="true|false (default: false)" + ) + irs.add_argument( + "--max-num-batched-tokens", type=int, default=None, dest="max_num_batched_tokens", help="default 12288" + ) + irs.add_argument( + "--cache-mode", dest="cache_mode", default=None, choices=("warm", "cold", "unknown"), help="methodology label" + ) # full-context descriptors (2026-06-07) so roofline cells pass publish_to_lake --strict irs.add_argument("--dataset", default=None, help="full-context: workload dataset (default random)") - irs.add_argument("--cudagraph-mode", dest="cudagraph_mode", default=None, help="full-context: full|piecewise|none|eager (the eager/cudagraph trap)") - irs.add_argument("--enforce-eager", dest="enforce_eager", type=lambda s: s.lower() in ("true", "1", "yes"), default=None, help="full-context: sets cudagraph_mode=eager when no explicit --cudagraph-mode") - irs.add_argument("--gpu-memory-utilization", dest="gpu_memory_utilization", type=float, default=None, help="full-context: gmu the sweep ran at") + irs.add_argument( + "--cudagraph-mode", + dest="cudagraph_mode", + default=None, + help="full-context: full|piecewise|none|eager (the eager/cudagraph trap)", + ) + irs.add_argument( + "--enforce-eager", + dest="enforce_eager", + type=lambda s: s.lower() in ("true", "1", "yes"), + default=None, + help="full-context: sets cudagraph_mode=eager when no explicit --cudagraph-mode", + ) + irs.add_argument( + "--gpu-memory-utilization", + dest="gpu_memory_utilization", + type=float, + default=None, + help="full-context: gmu the sweep ran at", + ) irs.add_argument("--image", default=None, help="full-context: serving image tag / vllm commit the sweep ran on") - irs.add_argument("--delivery", dest="delivery", default=None, choices=("image", "overlay", "patchedVllm", "infr-patch"), help="how code reached the cluster (delivery ladder) -- full-context descriptor") - irs.add_argument("--overlay-mode", dest="overlay_mode", default=None, choices=("subpath", "patchset-initcontainer", "pythonpath-sitecustomize"), help="overlay sub-tier when --delivery=overlay") - irs.add_argument("--patch-files", dest="patch_files", default=None, help="comma-separated patch files applied -- full-context descriptor") - irs.add_argument("--data-parallel", dest="data_parallel", type=int, default=None, help="full-context: DP replica count") - irs.add_argument("--pipeline-parallel", dest="pipeline_parallel", type=int, default=None, help="full-context: PP size") + irs.add_argument( + "--delivery", + dest="delivery", + default=None, + choices=("image", "overlay", "patchedVllm", "infr-patch"), + help="how code reached the cluster (delivery ladder) -- full-context descriptor", + ) + irs.add_argument( + "--overlay-mode", + dest="overlay_mode", + default=None, + choices=("subpath", "patchset-initcontainer", "pythonpath-sitecustomize"), + help="overlay sub-tier when --delivery=overlay", + ) + irs.add_argument( + "--patch-files", + dest="patch_files", + default=None, + help="comma-separated patch files applied -- full-context descriptor", + ) + irs.add_argument( + "--data-parallel", dest="data_parallel", type=int, default=None, help="full-context: DP replica count" + ) + irs.add_argument( + "--pipeline-parallel", dest="pipeline_parallel", type=int, default=None, help="full-context: PP size" + ) irs.add_argument("--dry-run", action="store_true", help="Parse + validate; do NOT write files") irs.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") irs.add_argument("--json", action="store_true", help="Emit JSON envelope") @@ -3345,25 +3553,104 @@ def build_parser() -> argparse.ArgumentParser: iva.add_argument("--model", required=True, help="Served model id (arm result.json carries no model)") iva.add_argument("--hardware", default=None, help="e.g. GB300, B200 (default: B200)") iva.add_argument("--quant", default=None, help="NVFP4 | FP8 | BF16 (default: NVFP4)") - iva.add_argument("--tensor-parallel", type=int, default=None, dest="tensor_parallel", help="TP size (default: arm result.json tp, else 8)") - iva.add_argument("--parallel-strategy", default=None, dest="parallel_strategy", choices=("TP", "EP"), help="TP or EP (default: TP)") - iva.add_argument("--mtp", type=lambda s: s.lower() in ("true", "1", "yes"), default=None, help="true|false (default: inferred from arm name)") - iva.add_argument("--max-num-batched-tokens", type=int, default=None, dest="max_num_batched_tokens", help="default 4096") - iva.add_argument("--cache-mode", dest="cache_mode", default=None, choices=("warm", "cold", "unknown"), help="methodology label (default: arm result.json warm flag)") + iva.add_argument( + "--tensor-parallel", + type=int, + default=None, + dest="tensor_parallel", + help="TP size (default: arm result.json tp, else 8)", + ) + iva.add_argument( + "--parallel-strategy", + default=None, + dest="parallel_strategy", + choices=("TP", "EP"), + help="TP or EP (default: TP)", + ) + iva.add_argument( + "--mtp", + type=lambda s: s.lower() in ("true", "1", "yes"), + default=None, + help="true|false (default: inferred from arm name)", + ) + iva.add_argument( + "--max-num-batched-tokens", type=int, default=None, dest="max_num_batched_tokens", help="default 4096" + ) + iva.add_argument( + "--cache-mode", + dest="cache_mode", + default=None, + choices=("warm", "cold", "unknown"), + help="methodology label (default: arm result.json warm flag)", + ) iva.add_argument("--notes", default=None, help="Free-form notes recorded on each cell") - iva.add_argument("--dataset", dest="dataset", default=None, help="Workload dataset (random|sharegpt|sonnet|aa|code|...) -- full-context descriptor") - iva.add_argument("--cudagraph-mode", dest="cudagraph_mode", default=None, help="full|piecewise|none|eager -- full-context descriptor") - iva.add_argument("--enforce-eager", dest="enforce_eager", type=lambda s: s.lower() in ("true", "1", "yes"), default=None, help="true|false -- sets cudagraph_mode=eager when no explicit --cudagraph-mode") - iva.add_argument("--gpu-memory-utilization", dest="gpu_memory_utilization", type=float, default=None, help="gmu the number was measured at -- full-context descriptor") - iva.add_argument("--kv-cache-dtype", dest="kv_cache_dtype", default=None, help="fp8_e4m3|bf16|nvfp4|... -- full-context descriptor") - iva.add_argument("--image", dest="image", default=None, help="serving image tag / vllm commit -- full-context descriptor") - iva.add_argument("--delivery", dest="delivery", default=None, choices=("image", "overlay", "patchedVllm", "infr-patch"), help="how code reached the cluster (delivery ladder) -- full-context descriptor") - iva.add_argument("--overlay-mode", dest="overlay_mode", default=None, choices=("subpath", "patchset-initcontainer", "pythonpath-sitecustomize"), help="overlay sub-tier when --delivery=overlay") - iva.add_argument("--patch-files", dest="patch_files", default=None, help="comma-separated patch files applied -- full-context descriptor") - iva.add_argument("--data-parallel", dest="data_parallel", type=int, default=None, help="DP replica count (default 1)") - iva.add_argument("--pipeline-parallel", dest="pipeline_parallel", type=int, default=None, help="PP size (default 1)") - iva.add_argument("--require-plot-ready", action="store_true", dest="require_plot_ready", - help="Hard-fail at import if any arm lacks Median TTFT / Request throughput (strict-publish fields)") + iva.add_argument( + "--dataset", + dest="dataset", + default=None, + help="Workload dataset (random|sharegpt|sonnet|aa|code|...) -- full-context descriptor", + ) + iva.add_argument( + "--cudagraph-mode", + dest="cudagraph_mode", + default=None, + help="full|piecewise|none|eager -- full-context descriptor", + ) + iva.add_argument( + "--enforce-eager", + dest="enforce_eager", + type=lambda s: s.lower() in ("true", "1", "yes"), + default=None, + help="true|false -- sets cudagraph_mode=eager when no explicit --cudagraph-mode", + ) + iva.add_argument( + "--gpu-memory-utilization", + dest="gpu_memory_utilization", + type=float, + default=None, + help="gmu the number was measured at -- full-context descriptor", + ) + iva.add_argument( + "--kv-cache-dtype", + dest="kv_cache_dtype", + default=None, + help="fp8_e4m3|bf16|nvfp4|... -- full-context descriptor", + ) + iva.add_argument( + "--image", dest="image", default=None, help="serving image tag / vllm commit -- full-context descriptor" + ) + iva.add_argument( + "--delivery", + dest="delivery", + default=None, + choices=("image", "overlay", "patchedVllm", "infr-patch"), + help="how code reached the cluster (delivery ladder) -- full-context descriptor", + ) + iva.add_argument( + "--overlay-mode", + dest="overlay_mode", + default=None, + choices=("subpath", "patchset-initcontainer", "pythonpath-sitecustomize"), + help="overlay sub-tier when --delivery=overlay", + ) + iva.add_argument( + "--patch-files", + dest="patch_files", + default=None, + help="comma-separated patch files applied -- full-context descriptor", + ) + iva.add_argument( + "--data-parallel", dest="data_parallel", type=int, default=None, help="DP replica count (default 1)" + ) + iva.add_argument( + "--pipeline-parallel", dest="pipeline_parallel", type=int, default=None, help="PP size (default 1)" + ) + iva.add_argument( + "--require-plot-ready", + action="store_true", + dest="require_plot_ready", + help="Hard-fail at import if any arm lacks Median TTFT / Request throughput (strict-publish fields)", + ) iva.add_argument("--dry-run", action="store_true", help="Parse + validate; do NOT write files") iva.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") iva.add_argument("--json", action="store_true", help="Emit JSON envelope") @@ -3482,12 +3769,16 @@ def build_parser() -> argparse.ArgumentParser: "kernel_reproducer_scaffold", description=CONTRACT["kernel_reproducer_scaffold"]["description"], ) - krs.add_argument("--kernel-name", required=True, dest="kernel_name", - help="Kernel task-impl template name, e.g. linear_sm100_mpk_task_impl") - krs.add_argument("--header", required=True, - help="Kernel header to #include, e.g. tasks/blackwell/linear_sm100_mpk.cuh") - krs.add_argument("--output-dir", required=True, dest="output_dir", - help="Directory to write the .cu + build script") + krs.add_argument( + "--kernel-name", + required=True, + dest="kernel_name", + help="Kernel task-impl template name, e.g. linear_sm100_mpk_task_impl", + ) + krs.add_argument( + "--header", required=True, help="Kernel header to #include, e.g. tasks/blackwell/linear_sm100_mpk.cuh" + ) + krs.add_argument("--output-dir", required=True, dest="output_dir", help="Directory to write the .cu + build script") krs.add_argument("--mma-m", type=int, default=128, dest="mma_m") krs.add_argument("--mma-n", type=int, default=16, dest="mma_n") krs.add_argument("--batch", type=int, default=8, help="BATCH_SIZE") @@ -3666,14 +3957,15 @@ def build_parser() -> argparse.ArgumentParser: "dcgm_grounded=false." ), ) - pl.add_argument("--dry-run", dest="dry_run", action="store_true", help="Build + write parquet locally; skip S3 upload.") + pl.add_argument( + "--dry-run", dest="dry_run", action="store_true", help="Build + write parquet locally; skip S3 upload." + ) pl.add_argument( "--i-understand-this-publishes-externally", dest="i_understand_this_publishes_externally", action="store_true", help=( - "Confirm current-turn approval to upload campaign data to external " - "S3 storage. Not required with --dry-run." + "Confirm current-turn approval to upload campaign data to external S3 storage. Not required with --dry-run." ), ) pl.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") @@ -3681,23 +3973,36 @@ def build_parser() -> argparse.ArgumentParser: pl.set_defaults(func=cmd_publish_to_lake) vv = sub.add_parser("value_view", description=CONTRACT["value_view"]["description"]) - vv.add_argument("--registry", default=None, - help="Path to value-findings.yaml (default: /../configs/value-findings.yaml)") + vv.add_argument( + "--registry", + default=None, + help="Path to value-findings.yaml (default: /../configs/value-findings.yaml)", + ) vv.add_argument("--out", default=None, help="Write the rendered markdown ledger to this path") - vv.add_argument("--format", choices=["table", "report"], default="table", - help="table = wide audit table (default); report = compact copy-paste summary for a report/Slack") + vv.add_argument( + "--format", + choices=["table", "report"], + default="table", + help="table = wide audit table (default); report = compact copy-paste summary for a report/Slack", + ) vv.add_argument("--title", default=None, help="Ledger title") - vv.add_argument("--gpu-hr", type=float, default=None, - help="$/GPU-hour for the GRIND FRONTIER $/1M-token economics " - "(override; else perf-tune-report/configs/cost.yaml, else 8.60 GB300)") + vv.add_argument( + "--gpu-hr", + type=float, + default=None, + help="$/GPU-hour for the GRIND FRONTIER $/1M-token economics " + "(override; else perf-tune-report/configs/cost.yaml, else 8.60 GB300)", + ) vv.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") vv.add_argument("--json", action="store_true", help="Emit JSON envelope") vv.set_defaults(func=cmd_value_view) - pv = sub.add_parser("portability_view", - description=CONTRACT["portability_view"]["description"]) - pv.add_argument("--registry", default=None, - help="Path to value-findings.yaml (default: /../configs/value-findings.yaml)") + pv = sub.add_parser("portability_view", description=CONTRACT["portability_view"]["description"]) + pv.add_argument( + "--registry", + default=None, + help="Path to value-findings.yaml (default: /../configs/value-findings.yaml)", + ) pv.add_argument("--out", default=None, help="Write the rendered matrix markdown to this path") pv.add_argument("--title", default=None, help="Matrix title") pv.add_argument("--campaigns-dir", default=None, help="Override the campaigns root") @@ -3707,31 +4012,79 @@ def build_parser() -> argparse.ArgumentParser: # champion_select: baseline vs top-X cross-engine champion selection cs = sub.add_parser("champion_select", description=CONTRACT["champion_select"]["description"]) cs.add_argument("--campaign", required=True, help="Campaign dir path OR slug") - cs.add_argument("--focus", default=None, choices=("throughput", "latency", "mixed"), - help="Champion metric focus (default: campaign config.focus, else throughput)") - cs.add_argument("--focus-c", type=int, default=None, dest="focus_c", - help="Concurrency at which the champion is selected (default: 1 latency / 32 throughput)") + cs.add_argument( + "--focus", + default=None, + choices=("throughput", "latency", "mixed"), + help="Champion metric focus (default: campaign config.focus, else throughput)", + ) + cs.add_argument( + "--focus-c", + type=int, + default=None, + dest="focus_c", + help="Concurrency at which the champion is selected (default: 1 latency / 32 throughput)", + ) cs.add_argument("--top", type=int, default=3, help="Number of top variants to keep (default 3)") - cs.add_argument("--baseline", default=None, help="Baseline cell_id (default: a *-base/*-baseline arm, vLLM-preferred)") - cs.add_argument("--metric", default=None, choices=("tok_s_gpu", "tpot"), - help="Override the ranking metric (default: derived from --focus)") - cs.add_argument("--slo-rel", type=float, default=1.10, dest="slo_rel", - help="TPOT SLO = baseline TPOT x this (throughput focus; default 1.10)") - cs.add_argument("--slo-abs-ms", type=float, default=None, dest="slo_abs_ms", - help="Absolute TPOT SLO ceiling in ms (overrides --slo-rel)") - cs.add_argument("--trials", type=int, default=None, - help="Trials/arm of the A/B (VERDICT variance gate needs >=3 + --same-node)") - cs.add_argument("--same-node", action="store_true", dest="same_node", - help="Assert the A/B arms ran same-node (VERDICT variance gate)") - cs.add_argument("--require-workloads", default=None, dest="require_workloads", - help="Comma list the VERDICT multi-workload gate requires (default: aa,sonnet,sharegpt,random,code)") - cs.add_argument("--workloads-present", default=None, dest="workloads_present", - help="Comma list of workloads actually benched (multi-workload gate; omit => unknown)") - cs.add_argument("--accuracy-gate", default="unknown", dest="accuracy_gate", - choices=("pass", "fail", "unknown"), help="Accuracy-eval gate result (VERDICT gate)") - cs.add_argument("--accuracy-floor", type=float, default=None, dest="accuracy_floor", - help="Derive the accuracy gate from the campaign's local eval_acc cells " - "(import_model_eval): pass iff the worst measured metric >= this floor") + cs.add_argument( + "--baseline", default=None, help="Baseline cell_id (default: a *-base/*-baseline arm, vLLM-preferred)" + ) + cs.add_argument( + "--metric", + default=None, + choices=("tok_s_gpu", "tpot"), + help="Override the ranking metric (default: derived from --focus)", + ) + cs.add_argument( + "--slo-rel", + type=float, + default=1.10, + dest="slo_rel", + help="TPOT SLO = baseline TPOT x this (throughput focus; default 1.10)", + ) + cs.add_argument( + "--slo-abs-ms", + type=float, + default=None, + dest="slo_abs_ms", + help="Absolute TPOT SLO ceiling in ms (overrides --slo-rel)", + ) + cs.add_argument( + "--trials", type=int, default=None, help="Trials/arm of the A/B (VERDICT variance gate needs >=3 + --same-node)" + ) + cs.add_argument( + "--same-node", + action="store_true", + dest="same_node", + help="Assert the A/B arms ran same-node (VERDICT variance gate)", + ) + cs.add_argument( + "--require-workloads", + default=None, + dest="require_workloads", + help="Comma list the VERDICT multi-workload gate requires (default: aa,sonnet,sharegpt,random,code)", + ) + cs.add_argument( + "--workloads-present", + default=None, + dest="workloads_present", + help="Comma list of workloads actually benched (multi-workload gate; omit => unknown)", + ) + cs.add_argument( + "--accuracy-gate", + default="unknown", + dest="accuracy_gate", + choices=("pass", "fail", "unknown"), + help="Accuracy-eval gate result (VERDICT gate)", + ) + cs.add_argument( + "--accuracy-floor", + type=float, + default=None, + dest="accuracy_floor", + help="Derive the accuracy gate from the campaign's local eval_acc cells " + "(import_model_eval): pass iff the worst measured metric >= this floor", + ) cs.add_argument("--out", default=None, help="CHAMPION.md output path (default: /CHAMPION.md)") cs.add_argument("--title", default=None, help="Report title") cs.add_argument("--dry-run", action="store_true", help="Compute + print; do NOT write files") @@ -3775,9 +4128,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: # Optional deps that only land with `install.sh --full` (the perf_tune_report + # leaderboard extras). A bare `claude plugin install` install does NOT pull # these, so report_render / report_smoke / publish_to_lake fail without them. -_RENDERER_OPTIONAL_DEPS = frozenset( - {"matplotlib", "pandas", "numpy", "pyarrow", "boto3", "tiktoken", "openpyxl"} -) +_RENDERER_OPTIONAL_DEPS = frozenset({"matplotlib", "pandas", "numpy", "pyarrow", "boto3", "tiktoken", "openpyxl"}) def _renderer_dep_hint(missing: str) -> str: diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/portability_view.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/portability_view.py index 3ab719a..9c5cae5 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/portability_view.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/portability_view.py @@ -11,6 +11,7 @@ model). Dependency-light: reads only the small registry dict (no pyarrow / matplotlib), so it runs in a minimal install. Added in profile-and-optimize (perf_tune_report portability_view verb). """ + from __future__ import annotations from typing import Any @@ -26,7 +27,7 @@ def collect_models(registry: dict) -> list[str]: models: set[str] = set() for f in registry.get("findings", []): for k in _MODEL_KEYS: - for m in (f.get(k) or []): + for m in f.get(k) or []: if isinstance(m, str) and m.strip(): models.add(m.strip()) return sorted(models) @@ -52,14 +53,16 @@ def build_portability(registry: dict) -> dict[str, Any]: for f in registry.get("findings", []): if not any(f.get(k) for k in _MODEL_KEYS): continue # a finding that names no model has nothing to place on the matrix - rows.append({ - "id": f.get("id", ""), - "title": f.get("title", ""), - "applies_to": str(f.get("applies_to", "") or ""), - "lifecycle": f.get("lifecycle", "?"), - "next_value": str(f.get("next_value", "med") or "med").lower(), - "states": {m: cell_state(f, m) for m in models}, - }) + rows.append( + { + "id": f.get("id", ""), + "title": f.get("title", ""), + "applies_to": str(f.get("applies_to", "") or ""), + "lifecycle": f.get("lifecycle", "?"), + "next_value": str(f.get("next_value", "med") or "med").lower(), + "states": {m: cell_state(f, m) for m in models}, + } + ) candidates: dict[str, list[str]] = {m: [] for m in models} for r in rows: for m in models: diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/provenance.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/provenance.py index a462cdf..be8b2af 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/provenance.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/provenance.py @@ -59,8 +59,8 @@ from __future__ import annotations import re -from typing import Any from pathlib import Path +from typing import Any SCHEMA = "experiment_provenance_v1" @@ -218,9 +218,7 @@ def validate(prov: dict[str, Any]) -> list[str]: problems.append("identity.run_id is required (the immutable join key)") status = ident.get("status") if status and status not in STATUS_VALUES: - problems.append( - f"identity.status {status!r} not in {STATUS_VALUES}" - ) + problems.append(f"identity.status {status!r} not in {STATUS_VALUES}") srcs = _sources(prov) if not srcs: problems.append("source: must list >=1 code-under-test entry (repo + commit)") @@ -231,14 +229,10 @@ def validate(prov: dict[str, Any]) -> list[str]: problems.append(f"source[{i}].commit is required (the actual source SHA)") deliv = s.get("delivery") if deliv and deliv not in DELIVERY_KINDS: - problems.append( - f"source[{i}].delivery {deliv!r} not in {DELIVERY_KINDS}" - ) + problems.append(f"source[{i}].delivery {deliv!r} not in {DELIVERY_KINDS}") omode = s.get("overlay_mode") if omode and omode not in OVERLAY_MODES: - problems.append( - f"source[{i}].overlay_mode {omode!r} not in {OVERLAY_MODES}" - ) + problems.append(f"source[{i}].overlay_mode {omode!r} not in {OVERLAY_MODES}") verd = prov.get("verdict") or {} tier = verd.get("tier") if tier and tier not in VERDICT_TIERS: @@ -313,9 +307,7 @@ def provenance_match_problems( cam_commit = str(cv.get("commit", "") or "") else: # flattened lake / provenance.json form cam_delivery = str(campaign_prov.get("delivery", "") or "") - cam_commit = str( - campaign_prov.get("vllm_commit") or campaign_prov.get("code_sha") or "" - ) + cam_commit = str(campaign_prov.get("vllm_commit") or campaign_prov.get("code_sha") or "") if not cam_delivery and not cam_commit: return [] # campaign provenance unknown -> nothing to assert (ungrounded check covers it) pfx = f"{label}: " if label else "" @@ -404,16 +396,18 @@ def source_links(prov: dict[str, Any] | None, registry: dict[str, Any] | None = patch = ", ".join(str(p) for p in patch_files) else: patch = str(patch_files) - out.append({ - "repo": repo, - "branch": branch, - "commit": commit, - "delivery": str(s.get("delivery", "") or ""), - "overlay_mode": str(s.get("overlay_mode", "") or ""), - "image": str(s.get("image", "") or ""), - "image_digest": str(s.get("image_digest", "") or ""), - "url": github_url(repo, branch, commit), - "purpose": str(reg.get("purpose", "") or ""), - "patch": patch, - }) + out.append( + { + "repo": repo, + "branch": branch, + "commit": commit, + "delivery": str(s.get("delivery", "") or ""), + "overlay_mode": str(s.get("overlay_mode", "") or ""), + "image": str(s.get("image", "") or ""), + "image_digest": str(s.get("image_digest", "") or ""), + "url": github_url(repo, branch, commit), + "purpose": str(reg.get("purpose", "") or ""), + "patch": patch, + } + ) return out diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/raw_bench_compare.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/raw_bench_compare.py index 4d64416..40b3725 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/raw_bench_compare.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/raw_bench_compare.py @@ -62,7 +62,6 @@ from pathlib import Path from typing import Any - _SWEEP_C_RE = re.compile(r"^sweep-c(\d+)\.txt$") @@ -119,9 +118,7 @@ def to_dict(self) -> dict[str, Any]: } -def _parse_sweep_file( - path: Path, *, concurrency: int | None = None -) -> dict[str, float | int | str] | None: +def _parse_sweep_file(path: Path, *, concurrency: int | None = None) -> dict[str, float | int | str] | None: """Parse one vllm-bench-serve output file's metrics. Reuses the regex set from ``importers.inference_perf_bench._REGEX`` so the parser @@ -196,18 +193,12 @@ def load_manifest(manifest_path: Path) -> tuple[dict[str, Any], list[BundleSpec] try: data = _yaml.safe_load(manifest_path.read_text()) except _yaml.YAMLError as e: - raise RawBenchCompareManifestMalformed( - manifest_path, reason=f"YAML parse error: {e}" - ) from e + raise RawBenchCompareManifestMalformed(manifest_path, reason=f"YAML parse error: {e}") from e if not isinstance(data, dict): - raise RawBenchCompareManifestMalformed( - manifest_path, reason="top-level not a mapping" - ) + raise RawBenchCompareManifestMalformed(manifest_path, reason="top-level not a mapping") bundles_in = data.get("bundles") if not isinstance(bundles_in, list) or not bundles_in: - raise RawBenchCompareManifestMalformed( - manifest_path, reason="bundles[] missing or empty" - ) + raise RawBenchCompareManifestMalformed(manifest_path, reason="bundles[] missing or empty") bundles_root_raw = data.get("bundles_root") if bundles_root_raw: @@ -220,9 +211,7 @@ def load_manifest(manifest_path: Path) -> tuple[dict[str, Any], list[BundleSpec] specs: list[BundleSpec] = [] for entry in bundles_in: if not isinstance(entry, dict): - raise RawBenchCompareManifestMalformed( - manifest_path, reason=f"bundle entry not a mapping: {entry!r}" - ) + raise RawBenchCompareManifestMalformed(manifest_path, reason=f"bundle entry not a mapping: {entry!r}") for required in ("glob", "label", "short"): if required not in entry: raise RawBenchCompareManifestMalformed( @@ -270,10 +259,7 @@ def render_comparison( n_with_data = sum(1 for s in specs if s.rows) if n_with_data == 0: - raise ValueError( - f"render_comparison: no bundles in {manifest_path} yielded " - f"parseable rows; PDF would be empty" - ) + raise ValueError(f"render_comparison: no bundles in {manifest_path} yielded parseable rows; PDF would be empty") # Lazy matplotlib import. import matplotlib @@ -288,11 +274,13 @@ def render_comparison( notes = meta.get("notes", "") baseline = next((s for s in specs if s.is_baseline and s.rows), None) - baseline_peak_tps = ( - max((row["output_tps"] for row in baseline.rows if "output_tps" in row), default=None) - if baseline - else None - ) + baseline_tps_values: list[float] = [] + if baseline: + for row in baseline.rows: + output_tps = row.get("output_tps") + if isinstance(output_tps, (int, float)): + baseline_tps_values.append(float(output_tps)) + baseline_peak_tps = max(baseline_tps_values, default=None) out_pdf.parent.mkdir(parents=True, exist_ok=True) peaks_summary: list[dict[str, Any]] = [] @@ -307,13 +295,21 @@ def render_comparison( ax.text(0.5, 0.78, sub, ha="center", fontsize=11) ax.text(0.5, 0.73, f"manifest: {manifest_path.name}", ha="center", fontsize=8, color="#666") ax.text( - 0.5, 0.65, + 0.5, + 0.65, f"{n_with_data} of {len(specs)} bundles produced parseable data", - ha="center", fontsize=10, + ha="center", + fontsize=10, ) if notes: ax.text( - 0.5, 0.55, notes, ha="center", va="top", fontsize=9, color="#444", + 0.5, + 0.55, + notes, + ha="center", + va="top", + fontsize=9, + color="#444", wrap=True, ) pdf.savefig(fig) @@ -321,7 +317,8 @@ def render_comparison( # Page 2: throughput vs concurrency _render_line_chart( - pdf, specs, + pdf, + specs, metric_key="output_tps", title=f"Output throughput vs concurrency ({campaign_name})", ylabel="Output tokens / sec", @@ -329,7 +326,8 @@ def render_comparison( ) # Page 3: TTFT vs concurrency _render_line_chart( - pdf, specs, + pdf, + specs, metric_key="ttft_med_ms", title="Median TTFT vs concurrency (lower = better)", ylabel="Median TTFT (ms)", @@ -338,7 +336,8 @@ def render_comparison( ) # Page 4: TPOT vs concurrency _render_line_chart( - pdf, specs, + pdf, + specs, metric_key="tpot_med_ms", title="Median TPOT vs concurrency (lower = better)", ylabel="Median TPOT (ms / token)", @@ -378,8 +377,14 @@ def _render_line_chart(pdf, specs, *, metric_key, title, ylabel, logx=False, log lw = 2.5 if spec.is_baseline else 1.8 ls = "--" if spec.is_baseline else "-" ax.plot( - xs, ys, color=spec.color, marker=spec.marker, label=spec.label, - linewidth=lw, linestyle=ls, markersize=8, + xs, + ys, + color=spec.color, + marker=spec.marker, + label=spec.label, + linewidth=lw, + linestyle=ls, + markersize=8, ) plotted += 1 ax.set_title(title, fontsize=12, fontweight="bold") @@ -394,10 +399,14 @@ def _render_line_chart(pdf, specs, *, metric_key, title, ylabel, logx=False, log ax.legend(loc="best", fontsize=8, framealpha=0.9) else: ax.text( - 0.5, 0.5, + 0.5, + 0.5, f"No bundles carry '{metric_key}' rows", - transform=ax.transAxes, ha="center", va="center", - fontsize=10, color="#aa3333", + transform=ax.transAxes, + ha="center", + va="center", + fontsize=10, + color="#aa3333", ) fig.tight_layout() pdf.savefig(fig) @@ -431,33 +440,37 @@ def _render_peak_bars(pdf, specs, baseline_peak_tps: float | None) -> list[dict[ "peak_output_tps": float(best["output_tps"]), "peak_c": int(best["c"]), "pct_vs_baseline": ( - ((best["output_tps"] / baseline_peak_tps) - 1.0) * 100.0 - if baseline_peak_tps - else None + ((best["output_tps"] / baseline_peak_tps) - 1.0) * 100.0 if baseline_peak_tps else None ), } ) if not peaks: ax.text( - 0.5, 0.5, + 0.5, + 0.5, "No bundle has output_tps rows; peak-bars chart skipped.", - transform=ax.transAxes, ha="center", va="center", color="#aa3333", + transform=ax.transAxes, + ha="center", + va="center", + color="#aa3333", ) else: bars = ax.bar(labels, peaks, color=colors, edgecolor="black", linewidth=0.8) - for bar, peak, pc in zip(bars, peaks, peak_cs): + for bar, peak, pc in zip(bars, peaks, peak_cs, strict=True): pct = ((peak / baseline_peak_tps) - 1.0) * 100.0 if baseline_peak_tps else 0.0 ax.text( bar.get_x() + bar.get_width() / 2, bar.get_height() * 1.01, f"{peak:.0f}\n@ c={pc}" + (f"\n{pct:+.0f}%" if baseline_peak_tps else ""), - ha="center", va="bottom", fontsize=9, + ha="center", + va="bottom", + fontsize=9, ) ax.set_ylim(0, max(peaks) * 1.18) ax.set_title( - "Peak output throughput per variant" - + (" (% gain vs baseline)" if baseline_peak_tps else ""), - fontsize=12, fontweight="bold", + "Peak output throughput per variant" + (" (% gain vs baseline)" if baseline_peak_tps else ""), + fontsize=12, + fontweight="bold", ) ax.set_ylabel("Peak output tokens / sec") ax.grid(True, axis="y", alpha=0.3, linestyle="--") @@ -484,11 +497,7 @@ def _render_summary_table(pdf, specs, baseline_peak_tps: float | None) -> None: rows_table.append([spec.short, spec.knob, "n/a", "n/a", "n/a"]) continue best = max(ranked, key=lambda r: r["output_tps"]) - pct = ( - f"{((best['output_tps'] / baseline_peak_tps) - 1.0) * 100.0:+.1f}%" - if baseline_peak_tps - else "n/a" - ) + pct = f"{((best['output_tps'] / baseline_peak_tps) - 1.0) * 100.0:+.1f}%" if baseline_peak_tps else "n/a" rows_table.append( [ spec.short, @@ -504,7 +513,8 @@ def _render_summary_table(pdf, specs, baseline_peak_tps: float | None) -> None: table.scale(1.0, 1.5) ax.set_title( "Summary table (per-variant peak + % gain vs baseline)", - fontsize=12, fontweight="bold", + fontsize=12, + fontweight="bold", ) fig.tight_layout() pdf.savefig(fig) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/champion_select.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/champion_select.py index a75635c..cce61bf 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/champion_select.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/champion_select.py @@ -63,17 +63,22 @@ def _fmt(v: Any, nd: int = 1) -> str: return str(v) -def _draw_roofline(axR, payload: dict[str, Any], ceilings: dict[str, Any] | None, - hardware_key: str | None) -> None: +def _draw_roofline(axR, payload: dict[str, Any], ceilings: dict[str, Any] | None, hardware_key: str | None) -> None: overlay = payload.get("roofline_overlay") or {} if not overlay or not ceilings or not hardware_key: axR.axis("off") - axR.text(0.5, 0.5, - "Roofline overlay unavailable\n(no roofline_sweep.json for the " - "selected variants,\nor no sol-ceilings.yaml for this hardware).\n" - "Capture and import a roofline bundle for each variant.", - ha="center", va="center", fontsize=8, color="#a33", - transform=axR.transAxes) + axR.text( + 0.5, + 0.5, + "Roofline overlay unavailable\n(no roofline_sweep.json for the " + "selected variants,\nor no sol-ceilings.yaml for this hardware).\n" + "Capture and import a roofline bundle for each variant.", + ha="center", + va="center", + fontsize=8, + color="#a33", + transform=axR.transAxes, + ) return hw = ceilings.get(hardware_key, {}) @@ -83,8 +88,13 @@ def _draw_roofline(axR, payload: dict[str, Any], ceilings: dict[str, Any] | None ridge = (comp_pf * 1e15) / (hbm_tb * 1e12) ai = [10 ** (i / 8) for i in range(0, 33)] - axR.plot(ai, [min(a * hbm_tb, comp_ceil) for a in ai], "k-", lw=2, - label=f"roofline ({comp_pf:.0f} PFLOPS / {hbm_tb:.0f} TB/s per GPU)") + axR.plot( + ai, + [min(a * hbm_tb, comp_ceil) for a in ai], + "k-", + lw=2, + label=f"roofline ({comp_pf:.0f} PFLOPS / {hbm_tb:.0f} TB/s per GPU)", + ) axR.axvline(ridge, color="#bbb", ls=":", lw=1) rec = payload.get("recommended_cell") @@ -101,9 +111,16 @@ def _draw_roofline(axR, payload: dict[str, Any], ceilings: dict[str, Any] | None dy.append(t * comp_ceil) dx.append((t / d) * ridge) if dx: - axR.scatter(dx, dy, s=90 if is_rec else 55, marker=mk, - edgecolor="k", linewidth=1.1 if is_rec else 0.4, - zorder=6 if is_rec else 5, label=f"decode {lab}") + axR.scatter( + dx, + dy, + s=90 if is_rec else 55, + marker=mk, + edgecolor="k", + linewidth=1.1 if is_rec else 0.4, + zorder=6 if is_rec else 5, + label=f"decode {lab}", + ) px, py = [], [] for pt in p.get("prefill", []): t, d = pt.get("tensor_active"), pt.get("dram_active") @@ -112,22 +129,23 @@ def _draw_roofline(axR, payload: dict[str, Any], ceilings: dict[str, Any] | None py.append(t * comp_ceil) px.append((t / d) * ridge) if px: - axR.scatter(px, py, s=95 if is_rec else 60, marker=mk, facecolor="none", - edgecolor="#d33", linewidth=1.2, zorder=6) + axR.scatter( + px, py, s=95 if is_rec else 60, marker=mk, facecolor="none", edgecolor="#d33", linewidth=1.2, zorder=6 + ) axR.set_xscale("log") axR.set_yscale("log") axR.set_xlabel("arithmetic intensity (FLOP/byte)", fontsize=8) axR.set_ylabel("achieved compute per GPU (TFLOP/s)", fontsize=8) - axR.set_title("Roofline overlay: baseline + all selected variants (per-GPU, DCGM)", - fontsize=9, loc="left") + axR.set_title("Roofline overlay: baseline + all selected variants (per-GPU, DCGM)", fontsize=9, loc="left") axR.grid(True, which="both", ls=":", alpha=0.35) axR.legend(fontsize=6, loc="lower right") axR.set_ylim(comp_ceil / 1e3, comp_ceil * 2) -def render_page(fig, payload: dict[str, Any], ceilings: dict[str, Any] | None = None, - hardware_key: str | None = None) -> None: +def render_page( + fig, payload: dict[str, Any], ceilings: dict[str, Any] | None = None, hardware_key: str | None = None +) -> None: """Draw the champion-selection page onto ``fig`` from a champion_select.json ``payload``. ``ceilings`` + ``hardware_key`` (optional) enable the roofline overlay panel; absent, the page renders the banner + table only.""" @@ -151,51 +169,77 @@ def render_page(fig, payload: dict[str, Any], ceilings: dict[str, Any] | None = # ---- header / banner ---- axHdr.axis("off") - axHdr.text(0.0, 0.92, "Production recommendation", fontsize=13, fontweight="bold", - va="top", transform=axHdr.transAxes) + axHdr.text( + 0.0, 0.92, "Production recommendation", fontsize=13, fontweight="bold", va="top", transform=axHdr.transAxes + ) banner_color = "#1a7a1a" if tier == "VERDICT" else "#b06a00" - axHdr.text(0.0, 0.60, f"RECOMMENDED: {rec} ({eng}) [{tier}]", - fontsize=12, fontweight="bold", color=banner_color, va="top", - transform=axHdr.transAxes) - axHdr.text(0.0, 0.30, - f"focus={payload.get('focus','?')} metric={metric_label} " - f"c={payload.get('focus_c','?')} hw={payload.get('hardware','?')} " - f"TP={payload.get('tensor_parallel','?')} " - f"SLO={_fmt(payload.get('slo_ms'))} ms baseline={payload.get('baseline_cell','?')}", - fontsize=8, color="#444", va="top", transform=axHdr.transAxes) - gates = payload.get("gates") or [] - gate_str = " ".join( - f"{g.get('name')}={g.get('status','?').upper()}" for g in gates + axHdr.text( + 0.0, + 0.60, + f"RECOMMENDED: {rec} ({eng}) [{tier}]", + fontsize=12, + fontweight="bold", + color=banner_color, + va="top", + transform=axHdr.transAxes, ) - axHdr.text(0.0, 0.06, f"gates: {gate_str}", fontsize=8, color="#333", - va="top", transform=axHdr.transAxes) + axHdr.text( + 0.0, + 0.30, + f"focus={payload.get('focus', '?')} metric={metric_label} " + f"c={payload.get('focus_c', '?')} hw={payload.get('hardware', '?')} " + f"TP={payload.get('tensor_parallel', '?')} " + f"SLO={_fmt(payload.get('slo_ms'))} ms baseline={payload.get('baseline_cell', '?')}", + fontsize=8, + color="#444", + va="top", + transform=axHdr.transAxes, + ) + gates = payload.get("gates") or [] + gate_str = " ".join(f"{g.get('name')}={g.get('status', '?').upper()}" for g in gates) + axHdr.text(0.0, 0.06, f"gates: {gate_str}", fontsize=8, color="#333", va="top", transform=axHdr.transAxes) # ---- table: baseline + top-X ---- axTbl.axis("off") - cols = ["variant", "engine", metric_label, "%win", "TPOT ms", "SLO", - "sol_rigor", "HBM%", "tensor%", "SM%", "roofline"] + cols = [ + "variant", + "engine", + metric_label, + "%win", + "TPOT ms", + "SLO", + "sol_rigor", + "HBM%", + "tensor%", + "SM%", + "roofline", + ] cell_text: list[list[str]] = [] row_colors: list[str] = [] for v in payload.get("variants", []): sol = v.get("sol") or {} is_base = v.get("is_baseline") is_rec = v.get("cell_id") == payload.get("recommended_cell") - win = "--" if is_base else ( - f"{v['pct_win_vs_baseline']:+.1f}%" if v.get("pct_win_vs_baseline") is not None else "n/a" + win = ( + "--" + if is_base + else (f"{v['pct_win_vs_baseline']:+.1f}%" if v.get("pct_win_vs_baseline") is not None else "n/a") + ) + cell_text.append( + [ + (v.get("cell_id", "?") + (" *" if is_rec else "")), + v.get("engine", "?"), + _fmt(v.get("focus_metric")), + win, + _fmt(v.get("tpot_median_ms")), + v.get("slo_verdict", "?"), + sol.get("sol_rigor", "none"), + _fmt(sol.get("hbm_pct_sol")), + _fmt(sol.get("tensor_pct_sol")), + _fmt(sol.get("sm_active_pct")), + "yes" if v.get("has_roofline") else "no", + ] ) - cell_text.append([ - (v.get("cell_id", "?") + (" *" if is_rec else "")), - v.get("engine", "?"), - _fmt(v.get("focus_metric")), - win, - _fmt(v.get("tpot_median_ms")), - v.get("slo_verdict", "?"), - sol.get("sol_rigor", "none"), - _fmt(sol.get("hbm_pct_sol")), - _fmt(sol.get("tensor_pct_sol")), - _fmt(sol.get("sm_active_pct")), - "yes" if v.get("has_roofline") else "no", - ]) row_colors.append("#d6f0d6" if is_rec else ("#eef3f8" if is_base else "#ffffff")) if cell_text: @@ -218,6 +262,5 @@ def render_page(fig, payload: dict[str, Any], ceilings: dict[str, Any] | None = note = _TIER_NOTE if reasons: note = "Not a VERDICT: " + "; ".join(reasons[:3]) + ". " + note - fig.text(0.5, 0.012, note, ha="center", va="bottom", fontsize=6.5, color="#666", - wrap=True) - fig.suptitle(f"Champion selection -- {payload.get('campaign_id','')}", fontsize=12, y=0.985) + fig.text(0.5, 0.012, note, ha="center", va="bottom", fontsize=6.5, color="#666", wrap=True) + fig.suptitle(f"Champion selection -- {payload.get('campaign_id', '')}", fontsize=12, y=0.985) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_category_attribution.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_category_attribution.py index 96d6d16..864f526 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_category_attribution.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_category_attribution.py @@ -32,7 +32,6 @@ from collections import OrderedDict from typing import Any - _CATEGORY_DISPLAY_ORDER: tuple[str, ...] = ( "NCCL", "MoE", @@ -76,7 +75,7 @@ def _fmt_flops(f: float | None) -> str: def render_page( fig, - cell_dcgm: "OrderedDict[str, dict[str, Any]]", + cell_dcgm: OrderedDict[str, dict[str, Any]], ) -> None: """Draw the per-category attribution page onto a matplotlib Figure. @@ -94,17 +93,12 @@ def render_page( payload = cell_dcgm[first_vid] attribution = payload.get("per_category_attribution") or [] if not attribution: - raise ValueError( - "dcgm_category_attribution.render_page: per_category_attribution is empty" - ) + raise ValueError("dcgm_category_attribution.render_page: per_category_attribution is empty") - import matplotlib.pyplot as plt import numpy as np from matplotlib import gridspec hw_key = payload.get("hw_key", "?") - sweep_start = payload.get("sweep_start_utc", "?") - sweep_end = payload.get("sweep_end_utc", "?") duration_s = payload.get("duration_s", 0) n_gpus = payload.get("n_gpus", 0) @@ -120,20 +114,32 @@ def render_page( ax_hdr = fig.add_subplot(gs[0, 0]) ax_hdr.axis("off") ax_hdr.text( - 0.5, 0.78, + 0.5, + 0.78, "DCGM x zymtrace Per-Category Attribution", - ha="center", va="center", fontsize=14, fontweight="bold", + ha="center", + va="center", + fontsize=14, + fontweight="bold", ) ax_hdr.text( - 0.5, 0.46, - f"variant: {first_vid} | hardware: {hw_key} | " - f"GPUs: {n_gpus} | sweep duration: {duration_s:.0f}s", - ha="center", va="center", fontsize=8, color="#555555", + 0.5, + 0.46, + f"variant: {first_vid} | hardware: {hw_key} | GPUs: {n_gpus} | sweep duration: {duration_s:.0f}s", + ha="center", + va="center", + fontsize=8, + color="#555555", ) ax_hdr.text( - 0.5, 0.18, + 0.5, + 0.18, "Measured DCGM workload bytes / FLOPS attributed to zymtrace categories via time-share weighting", - ha="center", va="center", fontsize=8, color="#777777", style="italic", + ha="center", + va="center", + fontsize=8, + color="#777777", + style="italic", ) # --------------------------------- grouped bar chart (time-share + %SoL) @@ -177,7 +183,8 @@ def render_page( ax.set_ylabel("percentage (%)", fontsize=8) ax.set_title( f"Time-share + Cross-attributed %SoL per Category (variant: {first_vid})", - fontsize=10, loc="left", + fontsize=10, + loc="left", ) ax.tick_params(axis="y", labelsize=7) ax.axhline(100, color="#cc3333", linestyle="--", linewidth=0.7, alpha=0.6) @@ -198,29 +205,46 @@ def render_page( x[i], -max(time_shares + sol_pcts, default=0) * 0.04, f"{detail} [{kind}]", - ha="center", va="top", fontsize=6.5, color="#444444", + ha="center", + va="top", + fontsize=6.5, + color="#444444", ) # ------------------------------------------------------------ caveat ax_cv = fig.add_subplot(gs[2, 0]) ax_cv.axis("off") ax_cv.text( - 0.5, 0.75, + 0.5, + 0.75, "How to read: time-share is the fraction of GPU time the category occupied " "(zymtrace samples). %SoL is the cross-attributed measured throughput " "during that window divided by the category's natural ceiling.", - ha="center", va="center", fontsize=7, color="#555555", wrap=True, + ha="center", + va="center", + fontsize=7, + color="#555555", + wrap=True, ) ax_cv.text( - 0.5, 0.42, + 0.5, + 0.42, "Caveat: cross-attribution assumes DCGM workload bytes/FLOPS distribute " "uniformly within each category's time window. For per-individual-kernel " "rigor see the L4 page (ncu scatter); for total workload truth see the L3 " "page (DCGM bars).", - ha="center", va="center", fontsize=7, style="italic", color="#666666", + ha="center", + va="center", + fontsize=7, + style="italic", + color="#666666", ) ax_cv.text( - 0.5, 0.10, + 0.5, + 0.10, "See AGENTS.md 'Speed-of-light framing' for the four-level rigor hierarchy.", - ha="center", va="center", fontsize=7, color="#777777", + ha="center", + va="center", + fontsize=7, + color="#777777", ) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_sol.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_sol.py index 6e90cca..3a210ad 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_sol.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/dcgm_sol.py @@ -36,15 +36,15 @@ def _fmt_throughput(res: dict[str, Any]) -> str: if res.get("measured_bytes_per_s") is not None: bps = float(res["measured_bytes_per_s"]) if bps >= 1e12: - return f"{bps/1e12:.2f} TB/s/GPU" + return f"{bps / 1e12:.2f} TB/s/GPU" elif bps >= 1e9: - return f"{bps/1e9:.1f} GB/s/GPU" + return f"{bps / 1e9:.1f} GB/s/GPU" else: - return f"{bps/1e6:.0f} MB/s/GPU" + return f"{bps / 1e6:.0f} MB/s/GPU" if res.get("measured_tflops_avg") is not None: tf = float(res["measured_tflops_avg"]) if tf >= 1000: - return f"{tf/1000:.2f} PFLOPS/GPU" + return f"{tf / 1000:.2f} PFLOPS/GPU" else: return f"{tf:.1f} TFLOPS/GPU" return "(no measurement)" @@ -61,7 +61,7 @@ def _fmt_peak(res: dict[str, Any]) -> str: def render_page( fig, - cell_dcgm: "OrderedDict[str, dict[str, Any]]", + cell_dcgm: OrderedDict[str, dict[str, Any]], ) -> None: """Draw the DCGM workload-level SoL page onto a matplotlib Figure. @@ -73,7 +73,6 @@ def render_page( if not cell_dcgm: raise ValueError("dcgm_sol.render_page: cell_dcgm is empty") - import matplotlib.pyplot as plt from matplotlib import gridspec # For simplicity, the first cell drives the layout; multi-cell @@ -112,8 +111,7 @@ def render_page( ax_hdr.text( 0.5, 0.47, - f"variant: {first_vid} | hardware: {hw_key} | " - f"GPUs: {n_gpus} | DCGM group: {group_level}", + f"variant: {first_vid} | hardware: {hw_key} | GPUs: {n_gpus} | DCGM group: {group_level}", ha="center", va="center", fontsize=8, @@ -149,10 +147,7 @@ def render_page( key=lambda r: (r.get("peak_per_gpu_units", "") != "TB/s", r.get("peak_key", "")), ) labels = [r.get("peak_key", "?") for r in ordered] - sol_values = [ - (r.get("sol_pct") if r.get("sol_pct") is not None else 0.0) - for r in ordered - ] + sol_values = [(r.get("sol_pct") if r.get("sol_pct") is not None else 0.0) for r in ordered] y_pos = list(range(len(labels))) bars = ax.barh(y_pos, sol_values, edgecolor="white", linewidth=0.5) @@ -170,7 +165,7 @@ def render_page( ax.axvline(100, color="#cc3333", linestyle="--", linewidth=0.7, alpha=0.7) ax.text(99, -0.6, "ceiling", color="#cc3333", fontsize=7, ha="right") - for bar, res in zip(bars, ordered): + for bar, res in zip(bars, ordered, strict=True): sol = res.get("sol_pct") sol_str = f"{sol:.1f}%" if sol is not None else "n/a" label = f"{sol_str} | {_fmt_throughput(res)} | {_fmt_peak(res)}" @@ -192,8 +187,7 @@ def render_page( ax_cv.text( 0.5, cav_y, - "Warning: short sweep < dcgm_config.min_sweep_seconds; " - "DCGM scrape granularity makes integration coarse.", + "Warning: short sweep < dcgm_config.min_sweep_seconds; DCGM scrape granularity makes integration coarse.", ha="center", va="center", fontsize=7, @@ -204,8 +198,7 @@ def render_page( ax_cv.text( 0.5, cav_y, - "DCGM_FI_PROF group not exported on this cluster; " - "falling back to DCGM_FI_DEV_* counter-tier (coarser).", + "DCGM_FI_PROF group not exported on this cluster; falling back to DCGM_FI_DEV_* counter-tier (coarser).", ha="center", va="center", fontsize=7, @@ -216,8 +209,7 @@ def render_page( ax_cv.text( 0.5, cav_y, - "DCGM exporter not detected on this cluster; " - "no byte-grounded measurements available.", + "DCGM exporter not detected on this cluster; no byte-grounded measurements available.", ha="center", va="center", fontsize=7, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/heatmap_tables.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/heatmap_tables.py index 32f7253..0e2a95c 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/heatmap_tables.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/heatmap_tables.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from tools.perf_tune_report.renderer.style import legend_groups_in_order, style_for from tools.perf_tune_report.schema import ( @@ -51,6 +51,7 @@ def _metrics_for(rows: Sequence[AtlasCell]) -> tuple[tuple[str, str, str], ...]: metrics.append((field, label, fmt)) return tuple(metrics) + GRAY_CELL_BG = "#dddddd" HEADER_BG = "#f3f3f3" @@ -69,8 +70,13 @@ def _index_atlas(rows: Sequence[AtlasCell]): def _cell_text_and_color( - by_lkc: dict, cell_status: dict, legend_key: tuple, mbt: int, concurrency: int, - metric: str, fmt: str, + by_lkc: dict, + cell_status: dict, + legend_key: tuple, + mbt: int, + concurrency: int, + metric: str, + fmt: str, ) -> tuple[str, str]: """Return (display_text, background_color) for one heatmap cell.""" row = by_lkc.get((legend_key, mbt, concurrency)) @@ -147,8 +153,13 @@ def render_page(fig, rows: Sequence[AtlasCell]) -> None: row_col = ["white"] for mbt in mbt_values: text, bg = _cell_text_and_color( - by_lkc, cell_status, variant_cell.legend_key, mbt, c, - metric, fmt, + by_lkc, + cell_status, + variant_cell.legend_key, + mbt, + c, + metric, + fmt, ) row_text.append(text) row_col.append(bg) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/kernel_breakdown.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/kernel_breakdown.py index 69e156e..0737f56 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/kernel_breakdown.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/kernel_breakdown.py @@ -29,8 +29,7 @@ from __future__ import annotations from collections import OrderedDict -from typing import Any, Sequence - +from typing import Any # Display order for the stacked bar's categories. Mirrors the bucketing # in the capture query and in @@ -51,7 +50,7 @@ def _truncate(s: str, n: int) -> str: return s if len(s) <= n else s[: n - 1] + "\u2026" -def render_page(fig, cell_kernels: "OrderedDict[str, dict[str, Any]]") -> None: +def render_page(fig, cell_kernels: OrderedDict[str, dict[str, Any]]) -> None: """Draw the kernel-breakdown page onto a matplotlib Figure. Args: @@ -63,7 +62,6 @@ def render_page(fig, cell_kernels: "OrderedDict[str, dict[str, Any]]") -> None: if not cell_kernels: raise ValueError("kernel_breakdown.render_page: cell_kernels is empty") - import matplotlib.pyplot as plt from matplotlib import gridspec # 3 vertical zones: header (small), upper plot, lower table block. @@ -91,8 +89,7 @@ def render_page(fig, cell_kernels: "OrderedDict[str, dict[str, Any]]") -> None: ax_header.text( 0.5, 0.15, - f"{len(cell_kernels)} variant(s) with kernels.json; " - "samples from zymtrace_profiling.events JOIN interp_funcs", + f"{len(cell_kernels)} variant(s) with kernels.json; samples from zymtrace_profiling.events JOIN interp_funcs", ha="center", va="center", fontsize=8, @@ -102,9 +99,7 @@ def render_page(fig, cell_kernels: "OrderedDict[str, dict[str, Any]]") -> None: # Zone 2: per-category stacked bar across variants. ax_cat = fig.add_subplot(gs[1, :]) variant_ids = list(cell_kernels.keys()) - cat_totals_per_variant: dict[str, list[float]] = { - cat: [] for cat in CATEGORY_DISPLAY_ORDER - } + cat_totals_per_variant: dict[str, list[float]] = {cat: [] for cat in CATEGORY_DISPLAY_ORDER} for vid in variant_ids: cats = cell_kernels[vid].get("per_category", {}) for cat in CATEGORY_DISPLAY_ORDER: @@ -123,7 +118,7 @@ def render_page(fig, cell_kernels: "OrderedDict[str, dict[str, Any]]") -> None: edgecolor="white", linewidth=0.4, ) - bottom = [b + v for b, v in zip(bottom, vals)] + bottom = [b + v for b, v in zip(bottom, vals, strict=True)] ax_cat.set_title("Per-Category CUDA Sample Share by Variant", fontsize=10) ax_cat.set_ylabel("samples", fontsize=8) ax_cat.tick_params(axis="x", rotation=20, labelsize=7) @@ -143,9 +138,7 @@ def render_page(fig, cell_kernels: "OrderedDict[str, dict[str, Any]]") -> None: ax_table = fig.add_subplot(gs[2, :]) ax_table.axis("off") - ax_table.set_title( - f"Top-20 Kernels (variant: {first_vid})", fontsize=10, loc="left" - ) + ax_table.set_title(f"Top-20 Kernels (variant: {first_vid})", fontsize=10, loc="left") if top_kernels: cell_text = [ [ @@ -194,10 +187,7 @@ def render_page(fig, cell_kernels: "OrderedDict[str, dict[str, Any]]") -> None: top_py = first.get("top_python_during_cuda", [])[:10] if top_py: tbl_py = ax_py.table( - cellText=[ - [_truncate(p["frame"], 70), f"{int(p['samples']):,}"] - for p in top_py - ], + cellText=[[_truncate(p["frame"], 70), f"{int(p['samples']):,}"] for p in top_py], colLabels=["python frame", "samples"], colWidths=[0.82, 0.18], cellLoc="left", diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/prefill_decode_roofline.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/prefill_decode_roofline.py index b9cb964..00fd378 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/prefill_decode_roofline.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/prefill_decode_roofline.py @@ -54,7 +54,7 @@ def _compute_peak(ceilings_hw: dict[str, Any], quant: str) -> tuple[float, float return float(comp) if comp else 15.0, float(hbm) if hbm else 8.0 -def _resolve_shape(payload: dict[str, Any]) -> "roofline_math.ModelShape | None": +def _resolve_shape(payload: dict[str, Any]) -> roofline_math.ModelShape | None: """The analytical ModelShape for this config: prefer the importer-embedded ``analytical_shape`` block (self-contained, captured from the in-pod config.json), else the registry lookup by served model name. None => the @@ -63,8 +63,8 @@ def _resolve_shape(payload: dict[str, Any]) -> "roofline_math.ModelShape | None" if isinstance(embedded, dict) and embedded.get("hidden_size"): try: return roofline_math.shape_from_dict(embedded) - except Exception: # noqa: BLE001 (malformed embed -> fall through to registry) - pass + except TypeError: + return roofline_math.shape_for_model(payload.get("model", "")) return roofline_math.shape_for_model(payload.get("model", "")) @@ -91,7 +91,7 @@ def _prefill_rate(pt: dict[str, Any]) -> float | None: def render_page( fig, - cell_roofline: "OrderedDict[str, dict[str, Any]]", + cell_roofline: OrderedDict[str, dict[str, Any]], ceilings: dict[str, Any], hardware_key: str, ) -> None: @@ -104,14 +104,12 @@ def render_page( if missing: raise RooflineSweepJsonMalformed(cid, f"missing fields: {missing}") - import matplotlib.pyplot as plt # noqa: E402 from matplotlib import gridspec hw = ceilings.get(hardware_key, {}) hw_name = hw.get("hw_name", hardware_key) - gs = gridspec.GridSpec(2, 2, width_ratios=[1.35, 1.0], height_ratios=[1, 1], - hspace=0.34, wspace=0.26, figure=fig) + gs = gridspec.GridSpec(2, 2, width_ratios=[1.35, 1.0], height_ratios=[1, 1], hspace=0.34, wspace=0.26, figure=fig) axR = fig.add_subplot(gs[:, 0]) axB = fig.add_subplot(gs[0, 1]) axC = fig.add_subplot(gs[1, 1]) @@ -125,8 +123,13 @@ def render_page( # ---- Panel A: per-GPU roofline ---- # memory diagonal in TFLOPS/GPU: y = AI * hbm_tb (the 1e12 cancels TB/s vs TFLOP/s) ai_axis = [10 ** (i / 8) for i in range(-8, 33)] # ~0.1 .. ~1e4 - axR.plot(ai_axis, [min(a * hbm_tb, comp_ceil) for a in ai_axis], "k-", lw=2, - label=f"roofline (per-GPU: {comp_pf:.0f} PFLOPS / {hbm_tb:.0f} TB/s)") + axR.plot( + ai_axis, + [min(a * hbm_tb, comp_ceil) for a in ai_axis], + "k-", + lw=2, + label=f"roofline (per-GPU: {comp_pf:.0f} PFLOPS / {hbm_tb:.0f} TB/s)", + ) axR.axvline(ridge, color="#bbb", ls=":", lw=1) axR.text(ridge * 1.05, comp_ceil * 0.35, f"ridge\nAI={ridge:.0f}", fontsize=7, color="#666") @@ -166,10 +169,11 @@ def _cfg_label(cid: str, tp: Any) -> str: x = (t / d) * ridge y = t * comp_ceil if x and y: - dx.append(x); dy.append(y); dc.append(c) + dx.append(x) + dy.append(y) + dc.append(c) if dx: - sc = axR.scatter(dx, dy, c=dc, cmap="viridis", s=70, marker=mk, - edgecolor="k", linewidth=0.4, zorder=5) + sc = axR.scatter(dx, dy, c=dc, cmap="viridis", s=70, marker=mk, edgecolor="k", linewidth=0.4, zorder=5) axR.plot(dx, dy, "-", color="#3b7", lw=0.8, alpha=0.5, zorder=4) if not colorbar_done: cb = fig.colorbar(sc, ax=axR, pad=0.01, fraction=0.045) @@ -190,22 +194,28 @@ def _cfg_label(cid: str, tp: Any) -> str: if not t or not d: continue any_proxy = True - px.append((t / d) * ridge); py.append(t * comp_ceil) + px.append((t / d) * ridge) + py.append(t * comp_ceil) if px: - axR.scatter(px, py, marker=mk, c="#d33", s=95, edgecolor="k", - linewidth=0.4, zorder=6, label=f"prefill {lab}") - - axR.set_xscale("log"); axR.set_yscale("log") - xlab = ("arithmetic intensity (FLOP/byte) [analytical, from config.json]" - if not any_proxy else - "arithmetic intensity (FLOP/byte) [analytical; * = DCGM-proxy fallback]") - ylab = ("achieved compute / GPU (TFLOP/s) [flop_per_token x measured tok/s]" - if not any_proxy else - "achieved compute / GPU (TFLOP/s)") + axR.scatter( + px, py, marker=mk, c="#d33", s=95, edgecolor="k", linewidth=0.4, zorder=6, label=f"prefill {lab}" + ) + + axR.set_xscale("log") + axR.set_yscale("log") + xlab = ( + "arithmetic intensity (FLOP/byte) [analytical, from config.json]" + if not any_proxy + else "arithmetic intensity (FLOP/byte) [analytical; * = DCGM-proxy fallback]" + ) + ylab = ( + "achieved compute / GPU (TFLOP/s) [flop_per_token x measured tok/s]" + if not any_proxy + else "achieved compute / GPU (TFLOP/s)" + ) axR.set_xlabel(xlab, fontsize=8) axR.set_ylabel(ylab, fontsize=8) - axR.set_title("Prefill + Decode Roofline (per-GPU; analytical AI x measured tok/s)", - fontsize=10, loc="left") + axR.set_title("Prefill + Decode Roofline (per-GPU; analytical AI x measured tok/s)", fontsize=10, loc="left") axR.grid(True, which="both", ls=":", alpha=0.35) axR.legend(fontsize=6.5, loc="lower right") axR.set_ylim(comp_ceil / 1e4, comp_ceil * 2) @@ -229,21 +239,29 @@ def _cfg_label(cid: str, tp: Any) -> str: c, rate = pt.get("c"), _decode_rate(pt) if not c or not rate: continue - union = (min(shape.n_routed_experts, shape.n_experts_per_tok * c) - if shape.is_moe else 0) + union = min(shape.n_routed_experts, shape.n_experts_per_tok * c) if shape.is_moe else 0 wb_per_tok = shape.active_weight_bytes(union, quant) / c kv_per_tok = shape.kv_bytes_per_token(_decode_ctx(pt), kvd) delivered = (wb_per_tok + kv_per_tok) * rate / tp # bytes/s per GPU - bg_cs.append(c); bg.append(delivered / (hbm_tb * 1e12) * 100) + bg_cs.append(c) + bg.append(delivered / (hbm_tb * 1e12) * 100) if bg_cs: - axB.plot(bg_cs, bg, "s--", lw=1.3, alpha=0.8, - label=f"delivered HBM-BW % (bytes x tok/s) {_cfg_label(cid, tp)}") + axB.plot( + bg_cs, + bg, + "s--", + lw=1.3, + alpha=0.8, + label=f"delivered HBM-BW % (bytes x tok/s) {_cfg_label(cid, tp)}", + ) axB.axhline(75, color="#d33", ls="--", lw=1.4, label="target 75%") - axB.set_xscale("log", base=2); axB.set_ylim(0, 100) + axB.set_xscale("log", base=2) + axB.set_ylim(0, 100) axB.set_xlabel("decode concurrency C", fontsize=8) axB.set_ylabel("HBM-BW utilization %", fontsize=8) axB.set_title("Q2: decode HBM-BW utilization vs concurrency", fontsize=9, loc="left") - axB.grid(True, ls=":", alpha=0.35); axB.legend(fontsize=6.0, loc="upper left") + axB.grid(True, ls=":", alpha=0.35) + axB.legend(fontsize=6.0, loc="upper left") # ---- Panel C: tensor + SM util vs concurrency (Q1) ---- peak_decode_tensor = 0.0 @@ -256,21 +274,24 @@ def _cfg_label(cid: str, tp: Any) -> str: axC.plot(cs, te, "s-", lw=1.8, label=f"tensor {_l}") axC.plot(cs, sm, "d:", lw=1.2, alpha=0.7, label=f"SM {_l}") peak_decode_tensor = max(peak_decode_tensor, max(te)) - axC.set_xscale("log", base=2); axC.set_ylim(0, 100) + axC.set_xscale("log", base=2) + axC.set_ylim(0, 100) axC.set_xlabel("decode concurrency C", fontsize=8) axC.set_ylabel("utilization %", fontsize=8) axC.set_title("Q1: compute (tensor) + SM utilization vs concurrency", fontsize=9, loc="left") - axC.grid(True, ls=":", alpha=0.35); axC.legend(fontsize=6, loc="upper left") + axC.grid(True, ls=":", alpha=0.35) + axC.legend(fontsize=6, loc="upper left") # ---- auto mechanism annotation (observation, not interpretation) ---- - note = ( - f"decode peak: tensor {peak_decode_tensor:.0f}% of compute, " - f"HBM {peak_decode_hbm:.0f}% (target 75%). " - + ("memory-bound: decode never saturates compute; prefill is the compute phase." - if peak_decode_tensor < 25 else - "decode engages compute -- check prefill vs decode split.") + note = f"decode peak: tensor {peak_decode_tensor:.0f}% of compute, HBM {peak_decode_hbm:.0f}% (target 75%). " + ( + "memory-bound: decode never saturates compute; prefill is the compute phase." + if peak_decode_tensor < 25 + else "decode engages compute -- check prefill vs decode split." ) fig.text(0.5, 0.012, note, ha="center", fontsize=7.5, color="#444", wrap=True) - fig.suptitle(f"Prefill/Decode Roofline -- {hw_name} (measured DCGM x analytical AI; " - f"{len(cell_roofline)} config(s))", fontsize=12, y=0.98) + fig.suptitle( + f"Prefill/Decode Roofline -- {hw_name} (measured DCGM x analytical AI; {len(cell_roofline)} config(s))", + fontsize=12, + y=0.98, + ) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_report.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_report.py index a8c4d3e..7e0883d 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_report.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_report.py @@ -36,9 +36,10 @@ import os import textwrap from collections import OrderedDict -from datetime import datetime, timezone +from collections.abc import Sequence +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Sequence +from typing import Any from tools.perf_tune_report.coverage import summarize from tools.perf_tune_report.helpers import resolve_operator_path @@ -57,7 +58,6 @@ from tools.perf_tune_report.renderer.render_status import OMISSION_REASONS, OmissionReason, RenderStatus from tools.perf_tune_report.schema import AtlasCell, read_jsonl - _NCU_REQUIRED_FIELDS = ( "schema_version", "captured_sources", @@ -98,14 +98,14 @@ def __init__(self, path: Path, reason: str): def discover_ncu_payloads( campaign_dir: Path, -) -> "OrderedDict[str, dict[str, Any]]": +) -> OrderedDict[str, dict[str, Any]]: """Walk ``/cells/*/ncu_kernels.json``, parse + validate. Same pattern as ``discover_kernels_payloads`` for the zymtrace kernels.json. Empty dict means no campaign cell has ncu coverage; the renderer skips page 5 silently. """ - out: "OrderedDict[str, dict[str, Any]]" = OrderedDict() + out: OrderedDict[str, dict[str, Any]] = OrderedDict() cells_dir = campaign_dir / "cells" if not cells_dir.is_dir(): return out @@ -119,22 +119,20 @@ def discover_ncu_payloads( raise NcuKernelsJsonMalformed(path, f"not valid JSON: {e}") from e missing = [f for f in _NCU_REQUIRED_FIELDS if f not in payload] if missing: - raise NcuKernelsJsonMalformed( - path, f"missing required fields: {missing}" - ) + raise NcuKernelsJsonMalformed(path, f"missing required fields: {missing}") out[cell_dir.name] = payload return out def discover_dcgm_payloads( campaign_dir: Path, -) -> "OrderedDict[str, dict[str, Any]]": +) -> OrderedDict[str, dict[str, Any]]: """Walk ``/cells/*/dcgm_correlation.json``, parse + validate. Empty dict means no campaign cell has DCGM coverage; the renderer skips page 6 silently. """ - out: "OrderedDict[str, dict[str, Any]]" = OrderedDict() + out: OrderedDict[str, dict[str, Any]] = OrderedDict() cells_dir = campaign_dir / "cells" if not cells_dir.is_dir(): return out @@ -148,20 +146,18 @@ def discover_dcgm_payloads( raise DcgmCorrelationJsonMalformed(path, f"not valid JSON: {e}") from e missing = [f for f in _DCGM_REQUIRED_FIELDS if f not in payload] if missing: - raise DcgmCorrelationJsonMalformed( - path, f"missing required fields: {missing}" - ) + raise DcgmCorrelationJsonMalformed(path, f"missing required fields: {missing}") out[cell_dir.name] = payload return out def discover_roofline_payloads( campaign_dir: Path, -) -> "OrderedDict[str, dict[str, Any]]": +) -> OrderedDict[str, dict[str, Any]]: """Walk ``/cells/*/roofline_sweep.json`` (written by the roofline_sweep importer). Empty dict -> the prefill/decode roofline page is skipped silently. Malformed JSON raises (no silent degradation).""" - out: "OrderedDict[str, dict[str, Any]]" = OrderedDict() + out: OrderedDict[str, dict[str, Any]] = OrderedDict() cells_dir = campaign_dir / "cells" if not cells_dir.is_dir(): return out @@ -172,9 +168,7 @@ def discover_roofline_payloads( try: payload = json.loads(path.read_text()) except json.JSONDecodeError as e: - raise prefill_decode_roofline.RooflineSweepJsonMalformed( - cell_dir.name, f"not valid JSON: {e}" - ) from e + raise prefill_decode_roofline.RooflineSweepJsonMalformed(cell_dir.name, f"not valid JSON: {e}") from e out[cell_dir.name] = payload return out @@ -219,7 +213,7 @@ def __init__(self, path: Path, reason: str): def discover_kernels_payloads( campaign_dir: Path, -) -> "OrderedDict[str, dict[str, Any]]": +) -> OrderedDict[str, dict[str, Any]]: """Walk ``/cells/*/kernels.json``, parse + validate each. Returns an ``OrderedDict`` keyed by cell_id (= cell directory name), @@ -229,7 +223,7 @@ def discover_kernels_payloads( Raises ``KernelsJsonMalformed`` if any ``kernels.json`` file exists but is unparseable or missing required fields. """ - out: "OrderedDict[str, dict[str, Any]]" = OrderedDict() + out: OrderedDict[str, dict[str, Any]] = OrderedDict() cells_dir = campaign_dir / "cells" if not cells_dir.is_dir(): return out @@ -243,9 +237,7 @@ def discover_kernels_payloads( raise KernelsJsonMalformed(kernels_path, f"not valid JSON: {e}") from e missing = [f for f in _KERNELS_REQUIRED_FIELDS if f not in payload] if missing: - raise KernelsJsonMalformed( - kernels_path, f"missing required fields: {missing}" - ) + raise KernelsJsonMalformed(kernels_path, f"missing required fields: {missing}") out[cell_dir.name] = payload return out @@ -259,15 +251,15 @@ def _is_roofline_shard(name: str) -> bool: def compute_per_arm_coverage( campaign_dir: Path, - rows: "Sequence[AtlasCell]", - cell_kernels: "dict[str, Any]", - cell_dcgm: "dict[str, Any]", - cell_roofline: "dict[str, Any]", + rows: Sequence[AtlasCell], + cell_kernels: dict[str, Any], + cell_dcgm: dict[str, Any], + cell_roofline: dict[str, Any], *, page4: bool, page6: bool, page7: bool, -) -> "tuple[int, int, list[str]]": +) -> tuple[int, int, list[str]]: """Per-arm Speed-of-Light coverage rollup -- the source of truth the publish gate, teardown hook, and sol-coverage audit all key on. @@ -287,19 +279,17 @@ def compute_per_arm_coverage( definition without reimplementing coverage logic. """ cells_dir = campaign_dir / "cells" - cell_dirs = ( - {p.name for p in cells_dir.iterdir() if p.is_dir()} - if cells_dir.is_dir() - else set() - ) + cell_dirs = {p.name for p in cells_dir.iterdir() if p.is_dir()} if cells_dir.is_dir() else set() atlas_arm_dirs: dict[str, str] = {} for r in rows: cell_id = getattr(r, "cell_id", None) if not cell_id or _is_roofline_shard(cell_id): continue arm = (getattr(r, "extra", {}) or {}).get("arm") - if isinstance(arm, str) and arm and ( - arm in cell_dirs or f"{arm}-decode" in cell_dirs or f"{arm}-prefill" in cell_dirs + if ( + isinstance(arm, str) + and arm + and (arm in cell_dirs or f"{arm}-decode" in cell_dirs or f"{arm}-prefill" in cell_dirs) ): atlas_arm_dirs.setdefault(cell_id, arm) else: @@ -338,7 +328,7 @@ def compute_per_arm_coverage( _SOURCE_REGISTRY_RELPATH = Path("perf-tune-report") / "configs" / "source-registry.yaml" -def discover_provenance(campaign_dir: Path) -> "dict[str, Any] | None": +def discover_provenance(campaign_dir: Path) -> dict[str, Any] | None: """Read the campaign's ``provenance.json`` (the experiment_provenance_v1 block copied in by ``campaign_init``). None when absent (older campaigns).""" p = campaign_dir / "provenance.json" @@ -350,7 +340,7 @@ def discover_provenance(campaign_dir: Path) -> "dict[str, Any] | None": return None -def discover_source_registry(campaign_dir: Path) -> "dict[str, Any] | None": +def discover_source_registry(campaign_dir: Path) -> dict[str, Any] | None: """Walk up for ``perf-tune-report/configs/source-registry.yaml`` (branch -> purpose). None when not found (the source page then omits purposes).""" cur = campaign_dir.resolve() @@ -359,6 +349,7 @@ def discover_source_registry(campaign_dir: Path) -> "dict[str, Any] | None": if cand.is_file(): try: import yaml + return yaml.safe_load(cand.read_text(encoding="utf-8")) except Exception: # noqa: BLE001 return None @@ -522,7 +513,7 @@ def _render_completeness_page(fig: Any, status: RenderStatus) -> None: ) -def _render_source_page(fig: Any, links: "list[dict[str, str]]", ident: dict[str, Any]) -> None: +def _render_source_page(fig: Any, links: list[dict[str, str]], ident: dict[str, Any]) -> None: """Draw the "Source under test" page: the EXACT code each roofline ran (vLLM/SGLang repo + branch + commit + delivery + infr patch), with a GitHub URL per entry. A roofline with no traceable source is not reproducible; this @@ -539,8 +530,9 @@ def _render_source_page(fig: Any, links: "list[dict[str, str]]", ident: dict[str deliv = link.get("delivery") or "image" lines.append(f"{link['repo'] or '(repo unset)'} [delivery: {deliv}]") if link.get("branch") or link.get("commit"): - lines.append(f" branch: {link.get('branch') or '(default)'} " - f"commit: {link.get('commit') or '(unpinned)'}") + lines.append( + f" branch: {link.get('branch') or '(default)'} commit: {link.get('commit') or '(unpinned)'}" + ) if link.get("url"): lines.append(f" {link['url']}") if link.get("purpose"): @@ -557,10 +549,17 @@ def _render_source_page(fig: Any, links: "list[dict[str, str]]", ident: dict[str "(no experiment_provenance_v1 block. Add source repository, " "commit, delivery, and image details to SOURCE.md.)" ) - fig.text(0.5, 0.95, "SOURCE UNDER TEST -- exact code each roofline ran", - ha="center", va="top", fontsize=13, color="#1a5276", weight="bold") - fig.text(0.04, 0.88, "\n".join(lines), ha="left", va="top", fontsize=9, - family="monospace", color="#333333") + fig.text( + 0.5, + 0.95, + "SOURCE UNDER TEST -- exact code each roofline ran", + ha="center", + va="top", + fontsize=13, + color="#1a5276", + weight="bold", + ) + fig.text(0.04, 0.88, "\n".join(lines), ha="left", va="top", fontsize=9, family="monospace", color="#333333") def build_pdf_provenance( @@ -578,7 +577,7 @@ def build_pdf_provenance( metadata and a page-1 footer) so consumers never have to trust the local mtime. All emitted timestamps are UTC. """ - rendered_utc = rendered_at.astimezone(timezone.utc) + rendered_utc = rendered_at.astimezone(UTC) rendered_iso = rendered_utc.strftime("%Y-%m-%dT%H:%M:%SZ") captured = sorted({r.captured_at for r in rows if r.captured_at}) if not captured: @@ -594,10 +593,7 @@ def build_pdf_provenance( ) infodict = { "Subject": f"perf-report campaign {campaign_id}", - "Keywords": ( - f"campaign={campaign_id} rendered_utc={rendered_iso} " - f"bench_window={bench_window}" - ), + "Keywords": (f"campaign={campaign_id} rendered_utc={rendered_iso} bench_window={bench_window}"), "CreationDate": rendered_utc, "ModDate": rendered_utc, } @@ -680,7 +676,7 @@ def render_report( # UTC provenance stamp (embedded in PDF metadata + a page-1 footer) so the # artifact self-documents its canonical UTC instant + run-id, and nobody # has to read the local-wall-clock OS file mtime. - provenance = build_pdf_provenance(campaign_dir.name, rows, datetime.now(timezone.utc)) + provenance = build_pdf_provenance(campaign_dir.name, rows, datetime.now(UTC)) # Discover SoL inputs only when page 3 is in scope (page 4 layers on # top of page 3's zymtrace per_category data). @@ -833,9 +829,7 @@ def render_report( # category-level points from DCGM per_category_attribution # when ncu kernels have all-null AI/tflops (e.g. --set=basic # capture). - page5 = sol_roofline_scatter.render_page( - fig5, cell_ncu, ceilings, hw_key, cell_dcgm=cell_dcgm - ) + page5 = sol_roofline_scatter.render_page(fig5, cell_ncu, ceilings, hw_key, cell_dcgm=cell_dcgm) status.rendered_pages.append("ncu SoL scatter (page 5)") sol_l4 = True pdf.savefig(fig5) @@ -885,7 +879,11 @@ def render_report( roofline_inputs = discover_sol_inputs(campaign_dir, rows) if roofline_inputs is None and cell_roofline: hwtok = next(iter(cell_roofline.values())).get("hardware") - fb = {"B200": "b200_sm100", "GB300": "gb300_nvl72", "H100": "h100_sxm"}.get(hwtok) + fb = ( + {"B200": "b200_sm100", "GB300": "gb300_nvl72", "H100": "h100_sxm"}.get(hwtok) + if isinstance(hwtok, str) + else None + ) roofline_inputs = discover_sol_inputs(campaign_dir, rows, fallback_hw_key=fb) if cell_roofline and roofline_inputs is not None: r_ceilings, r_hw = roofline_inputs @@ -909,7 +907,11 @@ def render_report( champ_ceilings, champ_hw = roofline_inputs else: hwtok = champion_payload.get("hardware") - fb = {"B200": "b200_sm100", "GB300": "gb300_nvl72", "H100": "h100_sxm"}.get(hwtok) + fb = ( + {"B200": "b200_sm100", "GB300": "gb300_nvl72", "H100": "h100_sxm"}.get(hwtok) + if isinstance(hwtok, str) + else None + ) champ_inputs = discover_sol_inputs(campaign_dir, rows, fallback_hw_key=fb) if champ_inputs is not None: champ_ceilings, champ_hw = champ_inputs @@ -927,6 +929,7 @@ def render_report( prov_block = discover_provenance(campaign_dir) if prov_block: from tools.perf_tune_report import provenance as _prov_mod + links = _prov_mod.source_links(prov_block, discover_source_registry(campaign_dir)) fig_src = plt.figure(figsize=page_size) _render_source_page(fig_src, links, prov_block.get("identity", {}) or {}) @@ -982,9 +985,7 @@ def render_report( # proxy-vs-tight distinction is a recorded field, never a publish blocker -- # a latency-bound/ncu-only/proxy run is a first-class published result. status.sol_complete = sol_l1 or sol_l3 or sol_l4 - status.sol_rigor = ( - "L4" if sol_l4 else "L3" if sol_l3 else "L1" if sol_l1 else "none" - ) + status.sol_rigor = "L4" if sol_l4 else "L3" if sol_l3 else "L1" if sol_l1 else "none" # Machine-readable sidecar consumed by publish_to_lake + the CLI. status_path = campaign_dir / "report_status.json" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_status.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_status.py index 813bd4d..61fd6f3 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_status.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/render_status.py @@ -98,8 +98,7 @@ def as_dict(self) -> dict[str, str]: "tpm_table": OmissionReason( page="TPM supported across hardware types", why=( - "No atlas row carries output_tps_per_gpu, so per-hardware " - "tokens-per-minute capacity cannot be rolled up." + "No atlas row carries output_tps_per_gpu, so per-hardware tokens-per-minute capacity cannot be rolled up." ), how_to_fix=( "Ensure the bench output includes 'Output token throughput " @@ -179,8 +178,7 @@ def as_dict(self) -> dict[str, str]: "dcgm_correlation.json with per_category_attribution was found." ), how_to_fix=( - "Re-capture ncu with --roofline-min or --set full, or add DCGM per_category " - "attribution, then re-render." + "Re-capture ncu with --roofline-min or --set full, or add DCGM per_category attribution, then re-render." ), ), } diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/scatter_grid.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/scatter_grid.py index 4135c55..b0513c4 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/scatter_grid.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/scatter_grid.py @@ -15,7 +15,8 @@ from __future__ import annotations from collections import defaultdict -from typing import Sequence +from collections.abc import Sequence +from typing import Literal from tools.perf_tune_report.coverage import CoverageSummary from tools.perf_tune_report.renderer.style import ( @@ -143,9 +144,7 @@ def render_page( note = coverage.note_line() if note: header_lines.append(note) - header_lines.append( - "TTFT vs Request Throughput | Interactivity vs Throughput per GPU" - ) + header_lines.append("TTFT vs Request Throughput | Interactivity vs Throughput per GPU") header_lines.append( "Point labels show selected profiling concurrencies " f"({', '.join(str(c) for c in LABEL_CONCURRENCIES)}). " @@ -169,6 +168,7 @@ def render_page( handles = [] for cell in legend_groups: style = style_for(cell) + linestyle: Literal[":", "-"] = ":" if cell.parallel_strategy == "TP" else "-" handles.append( Line2D( [0], @@ -177,7 +177,7 @@ def render_page( marker=style.marker, markerfacecolor=style.markerfacecolor, markeredgecolor=style.color, - linestyle=style.linestyle, + linestyle=linestyle, linewidth=1.1, markersize=6, label=style.label, @@ -206,9 +206,7 @@ def render_page( x_label="Request throughput avg (req/s)", y_label="TTFT avg (ms)", ) - ax_left.set_title( - f"max_num_batched_tokens {mbt}", fontsize=7.5, loc="left", pad=2 - ) + ax_left.set_title(f"max_num_batched_tokens {mbt}", fontsize=7.5, loc="left", pad=2) ax_right = fig.add_subplot(gs[r, 1]) _plot_pane( @@ -219,6 +217,4 @@ def render_page( x_label="Output throughput/GPU avg (tok/s/GPU)", y_label="Output throughput/user avg (tok/s/user)", ) - ax_right.set_title( - f"max_num_batched_tokens {mbt}", fontsize=7.5, loc="left", pad=2 - ) + ax_right.set_title(f"max_num_batched_tokens {mbt}", fontsize=7.5, loc="left", pad=2) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline.py index 292735b..6ba7390 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline.py @@ -41,9 +41,9 @@ from __future__ import annotations from collections import OrderedDict +from collections.abc import Sequence from pathlib import Path -from typing import Any, Sequence - +from typing import Any # Per-category mapping is loaded from sol-ceilings.yaml's # ``category_ceiling_map`` at runtime. This module-level constant is the @@ -199,7 +199,7 @@ def _format_ceiling(row: dict[str, Any]) -> str: def render_page( fig, - cell_kernels: "OrderedDict[str, dict[str, Any]]", + cell_kernels: OrderedDict[str, dict[str, Any]], rows: Sequence[Any], ceilings: dict[str, Any], hardware_key: str, @@ -227,7 +227,6 @@ def render_page( f"hardware key {hardware_key!r} not in ceilings", ) - import matplotlib.pyplot as plt from matplotlib import gridspec hw_data = ceilings[hardware_key] @@ -310,7 +309,7 @@ def render_page( ax_cat.tick_params(axis="x", labelsize=7) ax_cat.set_xlim(0, max(shares) * 1.50 if shares else 1.0) - for bar, share, clabel, blabel in zip(bars, shares, ceiling_labels, bound_labels): + for bar, share, clabel, blabel in zip(bars, shares, ceiling_labels, bound_labels, strict=True): ax_cat.text( share + max(shares) * 0.02, bar.get_y() + bar.get_height() / 2, @@ -341,8 +340,7 @@ def render_page( ax_wl.text( 0.5, 0.5, - "(no atlas rows carry output_tps_per_gpu; " - "workload-level SoL skipped)", + "(no atlas rows carry output_tps_per_gpu; workload-level SoL skipped)", ha="center", va="center", fontsize=9, @@ -390,8 +388,7 @@ def render_page( ax_cv.text( 0.5, 0.55, - "Caveat: %SoL derived from zymtrace sample-share is a time-share " - "proxy, not byte-traffic or FLOP measurement.", + "Caveat: %SoL derived from zymtrace sample-share is a time-share proxy, not byte-traffic or FLOP measurement.", ha="center", va="center", fontsize=7, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline_scatter.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline_scatter.py index 54ff408..d230b61 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline_scatter.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/sol_roofline_scatter.py @@ -53,7 +53,7 @@ class Page5Status: def _dcgm_fallback_available( - cell_dcgm: "OrderedDict[str, dict[str, Any]] | None", + cell_dcgm: OrderedDict[str, dict[str, Any]] | None, ) -> bool: """True iff ``_plot_dcgm_fallback`` would plot at least one point. @@ -146,7 +146,7 @@ def _peak_bandwidth_tbps(hw_data: dict[str, Any]) -> tuple[float, str]: def _plot_dcgm_fallback( ax, - cell_dcgm: "OrderedDict[str, dict[str, Any]]", + cell_dcgm: OrderedDict[str, dict[str, Any]], *, ridge_ai: float, peak_tflops: float, @@ -172,7 +172,7 @@ def _plot_dcgm_fallback( Returns True iff at least one point was plotted. """ plotted = 0 - for ci, (cell_id, payload) in enumerate(cell_dcgm.items()): + for ci, (_cell_id, payload) in enumerate(cell_dcgm.items()): per_cat = payload.get("per_category_attribution") or [] duration_s = float(payload.get("duration_s") or 0.0) if duration_s <= 0: @@ -244,10 +244,10 @@ def _plot_dcgm_fallback( def render_page( fig, - cell_ncu: "OrderedDict[str, dict[str, Any]]", + cell_ncu: OrderedDict[str, dict[str, Any]], ceilings: dict[str, Any], hardware_key: str, - cell_dcgm: "OrderedDict[str, dict[str, Any]] | None" = None, + cell_dcgm: OrderedDict[str, dict[str, Any]] | None = None, ) -> Page5Status: """Draw the SoL roofline scatter page onto a matplotlib Figure. @@ -270,11 +270,8 @@ def render_page( if not cell_ncu: raise ValueError("sol_roofline_scatter.render_page: cell_ncu is empty") if hardware_key not in ceilings: - raise ValueError( - f"sol_roofline_scatter.render_page: hardware_key {hardware_key!r} not in ceilings" - ) + raise ValueError(f"sol_roofline_scatter.render_page: hardware_key {hardware_key!r} not in ceilings") - import matplotlib.pyplot as plt from matplotlib import gridspec hw_data = ceilings[hardware_key] @@ -295,15 +292,12 @@ def render_page( # "dcgm" = workload-level DCGM byte/FLOP fallback (measured, not partial); # "empty" = nothing plottable. _has_real_ai = any( - (k.get("arithmetic_intensity_flops_per_byte") or 0) > 0 - and (k.get("achieved_tflops") or 0) > 0 + (k.get("arithmetic_intensity_flops_per_byte") or 0) > 0 and (k.get("achieved_tflops") or 0) > 0 for payload in cell_ncu.values() for k in payload.get("kernels", []) ) _has_solonly = (not _has_real_ai) and any( - (k.get("achieved_sm_pct_peak") or 0) > 0 - for payload in cell_ncu.values() - for k in payload.get("kernels", []) + (k.get("achieved_sm_pct_peak") or 0) > 0 for payload in cell_ncu.values() for k in payload.get("kernels", []) ) if _has_real_ai: page_mode = "ncu" @@ -387,7 +381,7 @@ def render_page( # Decide axis range from data + the ceilings. all_ai: list[float] = [] all_tflops: list[float] = [] - for vid, payload in cell_ncu.items(): + for _vid, payload in cell_ncu.items(): for k in payload.get("kernels", []): ai = k.get("arithmetic_intensity_flops_per_byte") tf = k.get("achieved_tflops") @@ -398,7 +392,7 @@ def render_page( # Reasonable defaults if some kernels lack data. ai_min = min(all_ai) / 5.0 if all_ai else 0.05 - ai_max = max(all_ai + [ridge_ai]) * 5.0 if (all_ai or ridge_ai) else 1000.0 + ai_max = max([*all_ai, ridge_ai]) * 5.0 if (all_ai or ridge_ai) else 1000.0 tflops_min = min(all_tflops) / 5.0 if all_tflops else 0.1 tflops_max = peak_tflops * 1.5 @@ -437,10 +431,7 @@ def render_page( sm_pct = k.get("achieved_sm_pct_peak") if sm_pct is None or sm_pct <= 0: continue - ceil_tflops = ( - _category_compute_ceiling_tflops(cat, ceilings, hardware_key) - or peak_tflops - ) + ceil_tflops = _category_compute_ceiling_tflops(cat, ceilings, hardware_key) or peak_tflops y_est = (sm_pct / 100.0) * ceil_tflops if y_est <= 0: continue @@ -533,7 +524,11 @@ def render_page( used_dcgm_fallback = False if plotted_count == 0 and sol_only_count == 0 and cell_dcgm: used_dcgm_fallback = _plot_dcgm_fallback( - ax, cell_dcgm, ridge_ai=ridge_ai, peak_tflops=peak_tflops, peak_tbps=peak_tbps, + ax, + cell_dcgm, + ridge_ai=ridge_ai, + peak_tflops=peak_tflops, + peak_tbps=peak_tbps, ) # When STILL no points (no ncu kernels + no %SoL-only + @@ -571,7 +566,7 @@ def render_page( # Legend (deduped by category, since we only labelled the first cell) handles, labels = ax.get_legend_handles_labels() - by_label = dict(zip(labels, handles)) + by_label = dict(zip(labels, handles, strict=True)) if by_label: ax.legend(by_label.values(), by_label.keys(), loc="lower right", fontsize=7, frameon=True) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/style.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/style.py index bc33844..1dcee31 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/style.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/style.py @@ -16,8 +16,8 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass -from typing import Sequence from tools.perf_tune_report.schema import AtlasCell @@ -29,12 +29,12 @@ # the four hw/TP combos the GLM-5.1 PDF uses; unknown combos fall back to a # deterministic gray so the figure still renders without crashing. DEFAULT_PALETTE: dict[tuple[str, int], str] = { - ("H100", 16): "#1f77b4", # blue - ("B200", 8): "#2ca02c", # green - ("GB300", 4): "#d62728", # red - ("GB300", 8): "#9467bd", # purple (NVL72 8-GPU variants) - ("B200", 4): "#ff7f0e", # orange - ("H100", 8): "#8c564b", # brown + ("H100", 16): "#1f77b4", # blue + ("B200", 8): "#2ca02c", # green + ("GB300", 4): "#d62728", # red + ("GB300", 8): "#9467bd", # purple (NVL72 8-GPU variants) + ("B200", 4): "#ff7f0e", # orange + ("H100", 8): "#8c564b", # brown } _FALLBACK_COLOR = "#7f7f7f" # neutral gray diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/tpm_table.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/tpm_table.py index 9b56b01..4f6fa86 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/tpm_table.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/renderer/tpm_table.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from tools.perf_tune_report.renderer.style import color_for from tools.perf_tune_report.schema import AtlasCell @@ -24,10 +24,17 @@ SLA_MISS_BG = "#fde8e8" _COL_LABELS = [ - "Variant", "Point", "Conc", - "Out TPM/GPU", "Out TPM/repl", "Out TPM/node", - "Tot TPM/GPU", "Tot TPM/repl", "Tot TPM/node", - "$/1M out", "$/1M tot", + "Variant", + "Point", + "Conc", + "Out TPM/GPU", + "Out TPM/repl", + "Out TPM/node", + "Tot TPM/GPU", + "Tot TPM/repl", + "Tot TPM/node", + "$/1M out", + "$/1M tot", ] _N_COLS = len(_COL_LABELS) @@ -86,11 +93,7 @@ def shape_label_problems(groups: Sequence[TpmGroup]) -> list[str]: shape (no smoothing)"). """ shapes = sorted( - { - (g.mean_isl, g.mean_osl) - for g in groups - if g.mean_isl is not None or g.mean_osl is not None - }, + {(g.mean_isl, g.mean_osl) for g in groups if g.mean_isl is not None or g.mean_osl is not None}, key=lambda s: (s[0] or 0.0, s[1] or 0.0), ) if len(shapes) <= 1: @@ -114,11 +117,7 @@ def _shape_caption(groups: Sequence[TpmGroup]) -> list[str]: if shape_label_problems(groups): bits.append("ISL/OSL: per-row (varies)") else: - shapes = { - (g.mean_isl, g.mean_osl) - for g in groups - if g.mean_isl is not None or g.mean_osl is not None - } + shapes = {(g.mean_isl, g.mean_osl) for g in groups if g.mean_isl is not None or g.mean_osl is not None} if shapes: isl, osl = next(iter(shapes)) if isl is not None: @@ -149,8 +148,7 @@ def render_page( power_by_cell = power_by_cell or {} if not summary.groups: - fig.text(0.5, 0.5, "No throughput-bearing atlas rows for TPM rollup.", - ha="center", va="center") + fig.text(0.5, 0.5, "No throughput-bearing atlas rows for TPM rollup.", ha="center", va="center") return False hardwares: list[str] = [] @@ -160,9 +158,14 @@ def render_page( n = len(hardwares) gs = fig.add_gridspec( - n + 1, 1, + n + 1, + 1, height_ratios=[0.5] + [1.0] * n, - hspace=0.55, left=0.04, right=0.98, top=0.95, bottom=0.04, + hspace=0.55, + left=0.04, + right=0.98, + top=0.95, + bottom=0.04, ) header_ax = fig.add_subplot(gs[0, 0]) @@ -174,13 +177,17 @@ def render_page( ) ctx = f"\nData source / shape: {summary.context_line}" if summary.context_line else "" header_ax.text( - 0.0, 1.0, + 0.0, + 1.0, "TPM supported across hardware types\n" "TPM = tok/s * 60. 'peak' = warm sweep best-case (NOT cold steady-state); " "'sla' = highest tok/s/GPU meeting the latency SLA. Total-TPM is n/a when " "the backend emits no total-token line.\n" f"{sla_line} | per-node basis = {summary.gpus_per_node} GPUs{ctx}", - fontsize=7.5, va="top", ha="left", linespacing=1.5, + fontsize=7.5, + va="top", + ha="left", + linespacing=1.5, ) for hi, hw in enumerate(hardwares, start=1): @@ -222,7 +229,7 @@ def render_page( tbl.set_fontsize(6.0) tbl.scale(1.0, 1.3) - for r_idx, (color, is_miss) in enumerate(zip(label_colors, miss_flags), start=1): + for r_idx, (color, is_miss) in enumerate(zip(label_colors, miss_flags, strict=True), start=1): tbl[(r_idx, 0)].set_text_props(color=color, weight="bold") if is_miss: for c_idx in range(len(_COL_LABELS)): diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/roofline_math.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/roofline_math.py index 263282c..c9dd280 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/roofline_math.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/roofline_math.py @@ -28,7 +28,7 @@ from __future__ import annotations -from dataclasses import dataclass, asdict +from dataclasses import asdict, dataclass from typing import Any # --------------------------------------------------------------------------- @@ -82,10 +82,10 @@ class ModelShape: num_layers: int num_attention_heads: int vocab_size: int - intermediate_size: int # dense-MLP intermediate (gated SwiGLU: 3 matrices) - num_kv_heads: int = 0 # GQA; 0 => MHA (== num_attention_heads) - head_dim: int = 0 # 0 => hidden_size / num_attention_heads - gated_mlp: bool = True # SwiGLU (gate+up+down=3) vs (up+down=2) + intermediate_size: int # dense-MLP intermediate (gated SwiGLU: 3 matrices) + num_kv_heads: int = 0 # GQA; 0 => MHA (== num_attention_heads) + head_dim: int = 0 # 0 => hidden_size / num_attention_heads + gated_mlp: bool = True # SwiGLU (gate+up+down=3) vs (up+down=2) # Architecture-specific extra attention params per layer (e.g. the GLM-DSA # "lightning indexer" q/k projections). Generic families leave this 0. extra_attn_params_per_layer: float = 0.0 @@ -181,16 +181,11 @@ def kv_elems_per_token_per_layer(self) -> float: """Cached KV elements per token per layer.""" if self.is_mla: return float(self.kv_lora_rank + self.qk_rope_head_dim) # latent - return float(2 * self._num_kv_heads() * self._head_dim()) # K + V + return float(2 * self._num_kv_heads() * self._head_dim()) # K + V def kv_bytes_per_token(self, ctx_len: int, kv_dtype: str = "fp8") -> float: """Bytes of KV read per generated token (full context, all layers).""" - return ( - ctx_len - * self.num_layers - * self.kv_elems_per_token_per_layer() - * kv_bytes_per_elem(kv_dtype) - ) + return ctx_len * self.num_layers * self.kv_elems_per_token_per_layer() * kv_bytes_per_elem(kv_dtype) @property def flop_per_token(self) -> float: @@ -201,9 +196,7 @@ def flop_per_token(self) -> float: return 2.0 * self.active_params(experts) # -- roofline operating-point intensities ---------------------------- - def decode_arithmetic_intensity( - self, concurrency: int, ctx_len: int, quant: str, kv_dtype: str = "fp8" - ) -> float: + def decode_arithmetic_intensity(self, concurrency: int, ctx_len: int, quant: str, kv_dtype: str = "fp8") -> float: """FLOP/byte for a decode step at batch=concurrency, context=ctx_len. Weight bytes amortize across the batch (the expert union ``min(n_routed, @@ -247,8 +240,7 @@ def to_summary(self, quant: str, kv_dtype: str = "fp8") -> dict[str, Any]: "kv_bytes_per_elem": kv_bytes_per_elem(kv_dtype), "quant": quant, "kv_dtype": kv_dtype, - "total_params_b": round(self.active_params( - self.n_routed_experts if self.is_moe else 0) / 1e9, 2), + "total_params_b": round(self.active_params(self.n_routed_experts if self.is_moe else 0) / 1e9, 2), } @@ -282,16 +274,13 @@ def from_hf_config(config: dict[str, Any], name: str = "") -> ModelShape: inter = int(_first(config, "intermediate_size", "ffn_dim", "n_inner", default=4 * H)) # MoE detection - n_routed = int(_first(config, "n_routed_experts", "num_experts", - "num_local_experts", "n_experts", default=0)) - n_per_tok = int(_first(config, "num_experts_per_tok", "num_experts_per_token", - "moe_topk", "n_experts_per_tok", default=0)) - moe_inter = int(_first(config, "moe_intermediate_size", "expert_intermediate_size", - default=0)) - n_shared = int(_first(config, "n_shared_experts", "num_shared_experts", - "moe_num_shared_experts", default=0)) - first_dense = int(_first(config, "first_k_dense_replace", "moe_layer_start_index", - default=0)) + n_routed = int(_first(config, "n_routed_experts", "num_experts", "num_local_experts", "n_experts", default=0)) + n_per_tok = int( + _first(config, "num_experts_per_tok", "num_experts_per_token", "moe_topk", "n_experts_per_tok", default=0) + ) + moe_inter = int(_first(config, "moe_intermediate_size", "expert_intermediate_size", default=0)) + n_shared = int(_first(config, "n_shared_experts", "num_shared_experts", "moe_num_shared_experts", default=0)) + first_dense = int(_first(config, "first_k_dense_replace", "moe_layer_start_index", default=0)) is_moe = n_routed > 0 and n_per_tok > 0 if is_moe and moe_inter == 0: moe_inter = inter # fall back to dense intermediate if expert size absent @@ -312,13 +301,26 @@ def from_hf_config(config: dict[str, Any], name: str = "") -> ModelShape: return ModelShape( name=name or str(_first(config, "_name_or_path", "model_type", default="model")), - hidden_size=H, num_layers=L, num_attention_heads=n_heads, num_kv_heads=n_kv, - head_dim=head_dim, vocab_size=vocab, intermediate_size=inter, gated_mlp=gated, - is_moe=is_moe, n_routed_experts=n_routed, n_experts_per_tok=n_per_tok, - moe_intermediate_size=moe_inter, n_shared_experts=n_shared, + hidden_size=H, + num_layers=L, + num_attention_heads=n_heads, + num_kv_heads=n_kv, + head_dim=head_dim, + vocab_size=vocab, + intermediate_size=inter, + gated_mlp=gated, + is_moe=is_moe, + n_routed_experts=n_routed, + n_experts_per_tok=n_per_tok, + moe_intermediate_size=moe_inter, + n_shared_experts=n_shared, first_k_dense_replace=first_dense, - is_mla=is_mla, kv_lora_rank=kv_lora, q_lora_rank=q_lora, - qk_nope_head_dim=qk_nope, qk_rope_head_dim=qk_rope, v_head_dim=v_head, + is_mla=is_mla, + kv_lora_rank=kv_lora, + q_lora_rank=q_lora, + qk_nope_head_dim=qk_nope, + qk_rope_head_dim=qk_rope, + v_head_dim=v_head, ) @@ -332,12 +334,25 @@ def from_hf_config(config: dict[str, Any], name: str = "") -> ModelShape: _REGISTRY: dict[str, ModelShape] = { # GLM-5.1 (GlmMoeDsaForCausalLM) -- the exemplar, exact from /work/model/config.json "glm-5.1": ModelShape( - name="zai-org/GLM-5.1", hidden_size=6144, num_layers=78, num_attention_heads=64, - vocab_size=154880, intermediate_size=12288, gated_mlp=True, - is_moe=True, n_routed_experts=256, n_experts_per_tok=8, moe_intermediate_size=2048, - n_shared_experts=1, first_k_dense_replace=3, - is_mla=True, kv_lora_rank=512, q_lora_rank=2048, - qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256, + name="zai-org/GLM-5.1", + hidden_size=6144, + num_layers=78, + num_attention_heads=64, + vocab_size=154880, + intermediate_size=12288, + gated_mlp=True, + is_moe=True, + n_routed_experts=256, + n_experts_per_tok=8, + moe_intermediate_size=2048, + n_shared_experts=1, + first_k_dense_replace=3, + is_mla=True, + kv_lora_rank=512, + q_lora_rank=2048, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + v_head_dim=256, # GLM-DSA lightning indexer: q/k proj over IDX_HEADS=32 x IDX_DIM=128 extra_attn_params_per_layer=2 * 6144 * (32 * 128), ), diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_bench.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_bench.py index b1ada9e..f368ac5 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_bench.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_bench.py @@ -17,6 +17,7 @@ from __future__ import annotations +import os import shlex import subprocess import time @@ -28,9 +29,14 @@ DEFAULT_CUSTOM_DATASET_TYPE, MODE_DATASET_REPLAY, MODE_SYNTHETIC, + REDACTED_API_KEY, + build_aiperf_auth_config, build_aiperf_command, generate_aa_dataset, normalize_outputs, + redact_aiperf_command, + validate_aiperf_command, + write_aiperf_auth_config, ) from tools.perf_tune_report.runners.common import ( CellConfig, @@ -75,6 +81,31 @@ def _wrap_kube(inner: list[str], kube: dict) -> list[str]: ] +def _wrap_kube_with_stdin_auth_config(inner: list[str], kube: dict) -> list[str]: + """Run in the pod with the key and secret-free config read from stdin.""" + return [ + "kubectl", + "exec", + "-i", + "-n", + kube["namespace"], + "--context", + kube["kube_context"], + kube["bench_pod"], + "--", + "sh", + "-c", + ( + 'umask 077; cfg=$(mktemp "${TMPDIR:-/tmp}/aiperf-auth.XXXXXX.json") || exit 1; ' + "trap 'rm -f \"$cfg\"' EXIT; " + "IFS= read -r OPENAI_API_KEY || exit 2; export OPENAI_API_KEY; " + 'cat > "$cfg" || exit 1; "$@" --config "$cfg"' + ), + "sh", + *inner, + ] + + def run_cell( cell: CellConfig, campaign_dir: Path, @@ -105,11 +136,12 @@ def run_cell( ) -> AaResult: """Run one AA-shape cell across the cell's concurrencies.""" if shape_name not in AA_SHAPES: - raise ValueError( - f"unknown AA shape {shape_name!r}; expected one of {sorted(AA_SHAPES)}" - ) + raise ValueError(f"unknown AA shape {shape_name!r}; expected one of {sorted(AA_SHAPES)}") shape = AA_SHAPES[shape_name] aiperf_cmd = list(aiperf_cmd) if aiperf_cmd else ["aiperf"] + validate_aiperf_command(aiperf_cmd) + if api_key and ("\n" in api_key or "\r" in api_key): + raise ValueError("API keys must not contain newline characters") cell_dir = campaign_dir / "cells" / cell.cell_id cell_dir.mkdir(parents=True, exist_ok=True) @@ -122,17 +154,16 @@ def run_cell( resolved_input_file = input_file if mode == MODE_DATASET_REPLAY and not resolved_input_file: if kube is not None: - raise ValueError( - "dataset-replay + kube requires a pre-staged in-pod --input-file" - ) + raise ValueError("dataset-replay + kube requires a pre-staged in-pod --input-file") dataset_path = cell_dir / "dataset" / f"{shape.name}.jsonl" if not dry_run: - dataset_info = generate_aa_dataset( - shape, dataset_count or request_count, dataset_path - ) + dataset_info = generate_aa_dataset(shape, dataset_count or request_count, dataset_path) resolved_input_file = str(dataset_path) - commands: list[list[str]] = [] + commands_dir = cell_dir / "commands" + commands_dir.mkdir(exist_ok=True) + runtime_commands: list[list[str]] = [] + auth_configs: list[str | None] = [] for c in cell.concurrencies: if kube is not None: artifact_dir = f"/tmp/aa-{cell.cell_id}-c{c}" @@ -148,7 +179,6 @@ def run_cell( endpoint_type=endpoint_type, concurrency=c, request_count=request_count, - api_key=api_key, tokenizer=tokenizer, tokenizer_trust_remote_code=tokenizer_trust_remote_code, mode=mode, @@ -156,12 +186,40 @@ def run_cell( custom_dataset_type=custom_dataset_type, extra_output_controls=extra_output_controls, ) - commands.append(_wrap_kube(inner, kube) if kube is not None else inner) + auth_config_text: str | None = None + if api_key: + auth_config_path = commands_dir / f"aa-c{c}.aiperf.json" + auth_config_text = write_aiperf_auth_config( + auth_config_path, + build_aiperf_auth_config( + shape, + model=model, + url=url, + output_artifact_dir=artifact_dir, + endpoint=endpoint, + endpoint_type=endpoint_type, + concurrency=c, + request_count=request_count, + mode=mode, + input_file=resolved_input_file, + custom_dataset_type=custom_dataset_type, + extra_output_controls=extra_output_controls, + ), + ) - (cell_dir / "commands").mkdir(exist_ok=True) - (cell_dir / "commands" / "aa-sweep.cmd").write_text( - "\n".join(shlex.join(c) for c in commands) + "\n" - ) + if kube is None: + runtime_commands.append([*inner, "--config", str(auth_config_path)] if api_key else inner) + elif api_key: + runtime_commands.append(_wrap_kube_with_stdin_auth_config(inner, kube)) + else: + runtime_commands.append(_wrap_kube(inner, kube)) + auth_configs.append(auth_config_text) + + # Credentials stay out of argv. Evidence files and result payloads receive + # independent, redacted copies as a second boundary. + evidence_commands = [redact_aiperf_command(command) for command in runtime_commands] + + (commands_dir / "aa-sweep.cmd").write_text("\n".join(shlex.join(command) for command in evidence_commands) + "\n") if dry_run: write_status_file(cell_dir, STATUS_FAILED) @@ -169,7 +227,7 @@ def run_cell( cell_dir=cell_dir, status="dry-run", row_count=0, - commands=commands, + commands=evidence_commands, dry_run=True, shape=shape.name, mode=mode, @@ -177,6 +235,28 @@ def run_cell( ) raw_dir.mkdir(exist_ok=True) + + def run_aiperf(command: list[str], auth_config: str | None): + kwargs: dict[str, object] = { + "capture_output": True, + "text": True, + "check": False, + } + if api_key: + if kube is None: + child_env = os.environ.copy() + child_env["OPENAI_API_KEY"] = api_key + kwargs["env"] = child_env + else: + if auth_config is None: + raise RuntimeError("missing AIPerf auth config for Kubernetes run") + kwargs["input"] = f"{api_key}\n{auth_config}" + return subprocess_runner(command, **kwargs) + + def redact_output(text: str | None) -> str: + output = text or "" + return output.replace(api_key, REDACTED_API_KEY) if api_key else output + # Spec-decode AL capture (kube mode only): bracket each per-concurrency # run with a /metrics scrape from the bench pod so AL / accept-rate land # at the (cell, concurrency) atlas row grain, first-class instead of via @@ -205,17 +285,17 @@ def run_cell( settle_log.append(f"prewarm shapes={prewarm_shapes} {'ok' if ok else 'FAILED'}") if ok and settle_s > 0: sleeper(settle_s) - if burn_in and commands: + if burn_in and runtime_commands: # Run-and-discard one pass of the first concurrency point. In kube # mode the in-pod artifact dir is simply overwritten by the recorded # run; locally the burn-in writes to raw/burnin, which the # normalizer skips (its dir name carries no concurrency suffix). if kube is None: first_dir = str(raw_dir / f"c{cell.concurrencies[0]}") - burn_cmd = [tok.replace(first_dir, str(raw_dir / "burnin")) for tok in commands[0]] + burn_cmd = [token.replace(first_dir, str(raw_dir / "burnin")) for token in runtime_commands[0]] else: - burn_cmd = commands[0] - proc = subprocess_runner(burn_cmd, capture_output=True, text=True, check=False) + burn_cmd = runtime_commands[0] + proc = run_aiperf(burn_cmd, auth_configs[0]) settle_log.append(f"burn-in c={cell.concurrencies[0]} exit={proc.returncode} (DISCARDED)") if settle_s > 0: sleeper(settle_s) @@ -226,15 +306,15 @@ def run_cell( stdout_chunks: list[str] = [] stderr_chunks: list[str] = [] exits: list[int] = [] - for i, (cmd, c) in enumerate(zip(commands, cell.concurrencies)): + for i, (cmd, c, auth_config) in enumerate(zip(runtime_commands, cell.concurrencies, auth_configs, strict=True)): if i > 0 and settle_s > 0 and (prewarm_shapes or burn_in): sleeper(settle_s) # inter-point settle (only when discipline is on) if spec_capture is not None: spec_capture.scrape(f"pre-c{c}") - proc = subprocess_runner(cmd, capture_output=True, text=True, check=False) + proc = run_aiperf(cmd, auth_config) exits.append(proc.returncode) - stdout_chunks.append(f"# concurrency={c} exit={proc.returncode}\n{proc.stdout or ''}") - stderr_chunks.append(f"# concurrency={c} exit={proc.returncode}\n{proc.stderr or ''}") + stdout_chunks.append(f"# concurrency={c} exit={proc.returncode}\n{redact_output(proc.stdout)}") + stderr_chunks.append(f"# concurrency={c} exit={proc.returncode}\n{redact_output(proc.stderr)}") if spec_capture is not None: spec_capture.scrape(f"post-c{c}") spec_capture.window(c) @@ -252,9 +332,7 @@ def run_cell( (cell_dir / "commands" / "aa-sweep.stdout").write_text("\n".join(stdout_chunks)) (cell_dir / "commands" / "aa-sweep.stderr").write_text("\n".join(stderr_chunks)) - (cell_dir / "commands" / "aa-sweep.exit").write_text( - "\n".join(str(e) for e in exits) + "\n" - ) + (cell_dir / "commands" / "aa-sweep.exit").write_text("\n".join(str(e) for e in exits) + "\n") if spec_capture is not None: spec_capture.finalize() @@ -269,7 +347,7 @@ def run_cell( cell_dir=cell_dir, status=status, row_count=len(rows), - commands=commands, + commands=evidence_commands, dry_run=False, shape=shape.name, mode=mode, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_workload.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_workload.py index 546b923..244c563 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_workload.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aa_workload.py @@ -64,6 +64,30 @@ MODE_DATASET_REPLAY = "dataset-replay" MODES = (MODE_SYNTHETIC, MODE_DATASET_REPLAY) +# Command receipts preserve the invocation shape without exposing credentials. +REDACTED_API_KEY = "" +API_KEY_FLAG_ERROR = ( + "custom AIPerf commands must not include --api-key. " + "Use the managed API-key option or its configured environment variable." +) + + +def validate_aiperf_command(command: list[str]) -> None: + """Reject custom commands that could put an API key back in argv.""" + if any("--api-key" in token for token in command): + raise ValueError(API_KEY_FLAG_ERROR) + + +def redact_aiperf_command(command: list[str]) -> list[str]: + """Return a copy with every AIPerf API-key value redacted.""" + redacted = list(command) + for index, token in enumerate(redacted): + if token == "--api-key" and index + 1 < len(redacted): + redacted[index + 1] = REDACTED_API_KEY + elif token.startswith("--api-key="): + redacted[index] = f"--api-key={REDACTED_API_KEY}" + return redacted + @dataclass(frozen=True) class AAShape: @@ -74,6 +98,90 @@ class AAShape: output_tokens: int +def build_aiperf_auth_config( + shape: AAShape, + *, + model: str, + url: str, + output_artifact_dir: str, + endpoint: str = "/v1/chat/completions", + endpoint_type: str = "chat", + concurrency: int = 1, + request_count: int = 10, + mode: str = MODE_SYNTHETIC, + input_file: str | None = None, + custom_dataset_type: str = DEFAULT_CUSTOM_DATASET_TYPE, + extra_output_controls: bool = True, +) -> dict: + """Build a complete AIPerf config with an environment-backed API key. + + AIPerf ``profile`` does not read ``OPENAI_API_KEY`` as a direct CLI + default. It does resolve ``${OPENAI_API_KEY}`` in config files. Keeping + that placeholder in this non-secret config lets the child environment + carry the real key without exposing it in argv or evidence receipts. + """ + if mode not in MODES: + raise ValueError(f"mode must be one of {MODES}, got {mode!r}") + if mode == MODE_DATASET_REPLAY and not input_file: + raise ValueError("dataset-replay mode requires input_file") + + dataset: dict + if mode == MODE_SYNTHETIC: + dataset = { + "type": "synthetic", + "entries": request_count, + "prompts": { + "isl": {"mean": shape.input_tokens, "stddev": 0}, + "osl": {"mean": shape.output_tokens, "stddev": 0}, + }, + } + else: + dataset = { + "type": "file", + "path": input_file, + "format": custom_dataset_type, + "entries": request_count, + "force_min_tokens": True, + } + + extra: dict[str, int | bool] = { + "temperature": AA_TEMPERATURE, + "top_p": AA_TOP_P, + } + if extra_output_controls: + extra.update({"min_tokens": shape.output_tokens, "ignore_eos": True}) + + return { + "schemaVersion": "2.0", + "benchmark": { + "model": model, + "endpoint": { + "url": url, + "type": endpoint_type, + "path": endpoint, + "api_key": "${OPENAI_API_KEY}", + "streaming": True, + "extra": extra, + }, + "dataset": dataset, + "phases": { + "type": "concurrency", + "concurrency": concurrency, + "requests": request_count, + }, + "artifacts": {"dir": output_artifact_dir}, + }, + } + + +def write_aiperf_auth_config(path: Path, config: dict) -> str: + """Write a deterministic, secret-free AIPerf config and return its text.""" + text = json.dumps(config, indent=2, sort_keys=True) + "\n" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return text + + # Canonical AA shapes (single source of truth). The standalone script and the # test drift-guard both assert against this table. Numbers per the AA # methodology "Workload Types" table: @@ -104,9 +212,7 @@ def count_tokens(text: str, encoding_name: str = O200K_ENCODING) -> int: try: import tiktoken except ImportError as exc: # pragma: no cover - exercised via fallback path - raise TokenizerUnavailable( - "tiktoken is not installed; cannot count o200k_base tokens" - ) from exc + raise TokenizerUnavailable("tiktoken is not installed; cannot count o200k_base tokens") from exc enc = tiktoken.get_encoding(encoding_name) return len(enc.encode(text)) @@ -191,10 +297,7 @@ def generate_aa_dataset( text, measured, used_tiktoken = generate_prompt_text( shape.input_tokens, encoding_name=encoding_name, use_tiktoken=use_tiktoken ) - rows = [ - {"text_input": text, "output_length": shape.output_tokens} - for _ in range(count) - ] + rows = [{"text_input": text, "output_length": shape.output_tokens} for _ in range(count)] with out_path.open("w", encoding="utf-8") as f: for row in rows: f.write(json.dumps(row)) @@ -222,7 +325,6 @@ def build_aiperf_command( endpoint_type: str = "chat", concurrency: int = 1, request_count: int = 10, - api_key: str | None = None, tokenizer: str | None = None, tokenizer_trust_remote_code: bool = True, mode: str = MODE_SYNTHETIC, @@ -232,18 +334,23 @@ def build_aiperf_command( ) -> list[str]: """Build one ``aiperf profile`` argv for one AA shape at one concurrency. + Authentication is intentionally absent from argv. The runner supplies + ``OPENAI_API_KEY`` only in the child process environment. + Mirrors the original ``repro_artificialanalysis.sh`` flag set. In ``synthetic`` mode AIPerf generates the prompt at the token mean; in ``dataset-replay`` mode it replays ``input_file`` exactly. The ``temperature:0`` / ``top_p:1`` and (optional) ``min_tokens`` / ``ignore_eos`` extra-inputs are applied in both modes. """ + validate_aiperf_command(aiperf_cmd) if mode not in MODES: raise ValueError(f"mode must be one of {MODES}, got {mode!r}") if mode == MODE_DATASET_REPLAY and not input_file: raise ValueError("dataset-replay mode requires input_file") - cmd = list(aiperf_cmd) + [ + cmd = [ + *aiperf_cmd, "profile", "--model", model, @@ -255,8 +362,6 @@ def build_aiperf_command( "--url", url, ] - if api_key: - cmd += ["--api-key", api_key] if tokenizer: cmd += ["--tokenizer", tokenizer] if tokenizer_trust_remote_code: diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aiperf_bench.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aiperf_bench.py index 6208f70..c4f3858 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aiperf_bench.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/aiperf_bench.py @@ -148,9 +148,7 @@ def _extract_metrics(report: dict) -> dict | None: for source in candidates: if not isinstance(source, dict): continue - ttft = _metric_value( - source, "time_to_first_token", "median_ttft_ms", "mean_ttft_ms", "ttft_avg_ms" - ) + ttft = _metric_value(source, "time_to_first_token", "median_ttft_ms", "mean_ttft_ms", "ttft_avg_ms") rt = _metric_value(source, "request_throughput", "request_throughput_avg") ot = _metric_value( source, @@ -165,9 +163,7 @@ def _extract_metrics(report: dict) -> dict | None: # TTFO = first non-reasoning/answer token. Capturing both stops the # under-reporting that hid MiniMax-M2.7's ~4s think phase behind a # 0.10s "TTFT" (minimax-aabench). reasoning_token_count quantifies the think. - ttfo = _metric_value( - source, "time_to_first_output_token", "median_ttfo_ms", "mean_ttfo_ms" - ) + ttfo = _metric_value(source, "time_to_first_output_token", "median_ttfo_ms", "mean_ttfo_ms") reasoning_tokens = _metric_value( source, "reasoning_token_count", "reasoning_tokens", "output_reasoning_token_count" ) @@ -286,9 +282,7 @@ def run_cell( commands.append(cmd) (cell_dir / "commands").mkdir(exist_ok=True) - (cell_dir / "commands" / "aiperf-sweep.cmd").write_text( - "\n".join(shlex.join(c) for c in commands) + "\n" - ) + (cell_dir / "commands" / "aiperf-sweep.cmd").write_text("\n".join(shlex.join(c) for c in commands) + "\n") if dry_run: write_status_file(cell_dir, STATUS_FAILED) @@ -305,7 +299,7 @@ def run_cell( stdout_chunks: list[str] = [] stderr_chunks: list[str] = [] exits: list[int] = [] - for cmd, c in zip(commands, cell.concurrencies): + for cmd, c in zip(commands, cell.concurrencies, strict=True): proc = subprocess_runner(cmd, capture_output=True, text=True, check=False) exits.append(proc.returncode) stdout_chunks.append(f"# concurrency={c} exit={proc.returncode}\n{proc.stdout or ''}") @@ -323,9 +317,7 @@ def run_cell( (cell_dir / "commands" / "aiperf-sweep.stdout").write_text("\n".join(stdout_chunks)) (cell_dir / "commands" / "aiperf-sweep.stderr").write_text("\n".join(stderr_chunks)) - (cell_dir / "commands" / "aiperf-sweep.exit").write_text( - "\n".join(str(e) for e in exits) + "\n" - ) + (cell_dir / "commands" / "aiperf-sweep.exit").write_text("\n".join(str(e) for e in exits) + "\n") rows, status = normalize_outputs(cell, raw_dir, cell_dir) write_normalized_json(cell_dir, rows) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/common.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/common.py index 6e06e67..8cb462f 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/common.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/common.py @@ -5,7 +5,7 @@ import dataclasses import json from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -36,7 +36,7 @@ class CellConfig: def utc_now_iso() -> str: - return datetime.now(timezone.utc).isoformat(timespec="seconds") + return datetime.now(UTC).isoformat(timespec="seconds") def cell_config_from_dict(d: dict) -> CellConfig: diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/spec_metrics.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/spec_metrics.py index ccbdfe0..7f1fb39 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/spec_metrics.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/spec_metrics.py @@ -35,9 +35,7 @@ "vllm:iteration_tokens", ) -_METRIC_RE = re.compile( - r"^(?Pvllm:spec_decode_[^\s{]+)(?:\{(?P[^}]*)\})?\s+(?P[-+0-9.eE]+)$" -) +_METRIC_RE = re.compile(r"^(?Pvllm:spec_decode_[^\s{]+)(?:\{(?P[^}]*)\})?\s+(?P[-+0-9.eE]+)$") _LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"') SPEC_METRICS_DIRNAME = "spec_metrics" @@ -91,9 +89,7 @@ def parse_spec_totals(text: str) -> dict: labels = dict(_LABEL_RE.findall(match.group("labels") or "")) pos = labels.get("position") if pos is not None: - accepted_per_pos[int(pos)] = ( - accepted_per_pos.get(int(pos), 0.0) + value - ) + accepted_per_pos[int(pos)] = accepted_per_pos.get(int(pos), 0.0) + value return {**totals, "accepted_per_pos": accepted_per_pos} @@ -109,11 +105,7 @@ def compute_spec_window(pre_text: str, post_text: str) -> dict | None: return None positions = sorted(set(pre["accepted_per_pos"]) | set(post["accepted_per_pos"])) per_pos_accept_rate = { - str(pos): ( - post["accepted_per_pos"].get(pos, 0.0) - - pre["accepted_per_pos"].get(pos, 0.0) - ) - / drafts + str(pos): (post["accepted_per_pos"].get(pos, 0.0) - pre["accepted_per_pos"].get(pos, 0.0)) / drafts for pos in positions } return { @@ -126,9 +118,7 @@ def compute_spec_window(pre_text: str, post_text: str) -> dict | None: } -def attach_windows_to_rows( - rows: list[AtlasCell], windows: dict[int, dict] -) -> list[AtlasCell]: +def attach_windows_to_rows(rows: list[AtlasCell], windows: dict[int, dict]) -> list[AtlasCell]: """Set ``acceptance_length`` / ``spec_accept_rate`` on each row from its concurrency's window; raw counter deltas go to ``extra`` for provenance. Rows whose concurrency has no window pass through unchanged.""" @@ -179,18 +169,13 @@ def scrape(self, tag: str) -> bool: returns False (and logs) on failure instead of raising.""" self.dir.mkdir(parents=True, exist_ok=True) try: - proc = self._runner( - self._command, capture_output=True, text=True, check=False - ) + proc = self._runner(self._command, capture_output=True, text=True, check=False) except OSError as exc: self._log.append(f"{tag} FAILED exec: {exc}") return False if proc.returncode != 0 or not (proc.stdout or "").strip(): err = (proc.stderr or "").strip().splitlines() - self._log.append( - f"{tag} FAILED exit={proc.returncode}" - + (f" stderr={err[-1]}" if err else "") - ) + self._log.append(f"{tag} FAILED exit={proc.returncode}" + (f" stderr={err[-1]}" if err else "")) return False (self.dir / f"metrics-{tag}.prom").write_text(proc.stdout) self._texts[tag] = proc.stdout diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/vllm_sweep.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/vllm_sweep.py index 46c69fa..67b5793 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/vllm_sweep.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/runners/vllm_sweep.py @@ -84,10 +84,12 @@ def build_command( # Honor the cell's K so the SERVED config matches the RECORDED num_speculative_tokens # (was hardcoded to 1, which silently served K=1 even when the cell asked for K=2/3). _spx_spec = (cell.extras.get("serve_params_extra", {}) or {}).get("speculative_config") or {} - _k = (_spx_spec.get("num_speculative_tokens") - or cell.extras.get("num_speculative_tokens") - or cell.extras.get("spec_decode_k") - or 1) + _k = ( + _spx_spec.get("num_speculative_tokens") + or cell.extras.get("num_speculative_tokens") + or cell.extras.get("spec_decode_k") + or 1 + ) serve_params[0]["speculative_config"] = {"method": "mtp", "num_speculative_tokens": int(_k)} output_dir.mkdir(parents=True, exist_ok=True) serve_params_file.write_text(json.dumps(serve_params, indent=2) + "\n") @@ -180,11 +182,7 @@ def normalize_outputs(cell: CellConfig, raw_dir: Path, cell_dir: Path) -> tuple[ # Be lenient about the upstream schema -- different vLLM versions # emit slightly different field sets. We only need 5 fields. try: - concurrency = int( - data.get("max_concurrency") - or data.get("max_concurrent_requests") - or 0 - ) + concurrency = int(data.get("max_concurrency") or data.get("max_concurrent_requests") or 0) except (TypeError, ValueError): concurrency = 0 if concurrency <= 0: @@ -230,9 +228,7 @@ def normalize_outputs(cell: CellConfig, raw_dir: Path, cell_dir: Path) -> tuple[ else: status = STATUS_FULL # Stamp the resolved status onto each row. - rows = [ - AtlasCell(**{**r.to_dict(), "status": status}) for r in rows - ] + rows = [AtlasCell(**{**r.to_dict(), "status": status}) for r in rows] return rows, status @@ -250,14 +246,10 @@ def run_cell( cell_dir.mkdir(parents=True, exist_ok=True) write_backend_file(cell_dir, BACKEND_VLLM_SWEEP) - cmd = build_command( - cell, bench_cmd=bench_cmd, serve_cmd=serve_cmd, output_dir=cell_dir - ) + cmd = build_command(cell, bench_cmd=bench_cmd, serve_cmd=serve_cmd, output_dir=cell_dir) (cell_dir / "commands").mkdir(exist_ok=True) - (cell_dir / "commands" / "vllm-sweep.cmd").write_text( - shlex.join(cmd) + "\n" - ) + (cell_dir / "commands" / "vllm-sweep.cmd").write_text(shlex.join(cmd) + "\n") if dry_run: write_status_file(cell_dir, STATUS_FAILED) # not yet run diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/schema.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/schema.py index 79b4d8a..edec82a 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/schema.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/schema.py @@ -18,9 +18,10 @@ from __future__ import annotations import json +from collections.abc import Iterable from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import IO, Any, Iterable +from typing import IO, Any STATUS_FULL = "full" STATUS_PARTIAL = "partial" @@ -29,9 +30,9 @@ STATUSES = frozenset({STATUS_FULL, STATUS_PARTIAL, STATUS_FAILED, STATUS_EVICTED}) BACKEND_VLLM_SWEEP = "vllm-sweep" -BACKEND_SGLANG_SWEEP = "sglang-sweep" # cross-engine A/B (SGLang arm), same client/parser as vllm-sweep +BACKEND_SGLANG_SWEEP = "sglang-sweep" # cross-engine A/B (SGLang arm), same client/parser as vllm-sweep BACKEND_AIPERF = "aiperf" -BACKEND_TRTLLM = "trtllm" # Stub backend, not yet implemented +BACKEND_TRTLLM = "trtllm" # Stub backend, not yet implemented BACKENDS = frozenset({BACKEND_VLLM_SWEEP, BACKEND_SGLANG_SWEEP, BACKEND_AIPERF, BACKEND_TRTLLM}) # serving_engine is the normalized serving engine the bench exercised. The @@ -149,30 +150,30 @@ class AtlasCell: # and docs/METHODOLOGY.md. # All defaulted so older JSONL still parses; the methodology gate flags any # "unknown"/None on a MEASURED row (fail-closed under publish/render --strict). - dataset: str = "unknown" # random | sharegpt | sonnet | aa | code | ... - cudagraph_mode: str = "unknown" # full | piecewise | none | eager (the eager/cudagraph trap) + dataset: str = "unknown" # random | sharegpt | sonnet | aa | code | ... + cudagraph_mode: str = "unknown" # full | piecewise | none | eager (the eager/cudagraph trap) gpu_memory_utilization: float | None = None - kv_cache_dtype: str = "unknown" # fp8_e4m3 | bf16 | nvfp4 | ... - image: str = "unknown" # serving image tag / vllm commit (the stack the number is from) - data_parallel: int = 1 # DP replica count (1 = single instance) - pipeline_parallel: int = 1 # PP size (1 = none) + kv_cache_dtype: str = "unknown" # fp8_e4m3 | bf16 | nvfp4 | ... + image: str = "unknown" # serving image tag / vllm commit (the stack the number is from) + data_parallel: int = 1 # DP replica count (1 = single instance) + pipeline_parallel: int = 1 # PP size (1 = none) # Serving-variant descriptor: the per-variant knobs needed to answer # "which variant + why" and to build a stable cross-campaign variant_key. All # nullable/optional so older JSONL still parses AND so they never fail-close the # methodology gate (they are not always known/applicable on every run). - num_speculative_tokens: int | None = None # MTP / EAGLE K (the ``mtp`` bool is on/off; this is the K value) - async_scheduling: bool | None = None # --async-scheduling (host/decode overlap) - max_num_seqs: int | None = None # --max-num-seqs (decode batch cap; was extra-only) - enable_prefix_caching: bool | None = None # prefix-caching on/off (distinct from prefix_cache_hit_rate) - bench_backend: str = "" # bench CLIENT (vllm | openai), distinct from `backend` (the sweep runner) - variant_key: str = "" # stable serving-variant hash (capture_signature); populated at publish/aggregate + num_speculative_tokens: int | None = None # MTP / EAGLE K (the ``mtp`` bool is on/off; this is the K value) + async_scheduling: bool | None = None # --async-scheduling (host/decode overlap) + max_num_seqs: int | None = None # --max-num-seqs (decode batch cap; was extra-only) + enable_prefix_caching: bool | None = None # prefix-caching on/off (distinct from prefix_cache_hit_rate) + bench_backend: str = "" # bench CLIENT (vllm | openai), distinct from `backend` (the sweep runner) + variant_key: str = "" # stable serving-variant hash (capture_signature); populated at publish/aggregate # Ledger-to-atlas fields are nullable/defaulted so older JSONL parses. They # provide first-class capture for value-findings dimensions previously # carried only in ``extra``. None/"" means the lever does not apply. # Routing / load-balancer A/B descriptor (the KV-router findings). - router_policy: str = "" # round-robin | prefix-affinity | cache_aware | hybrid | adaptive | "" - prefix_reuse: float | None = None # prefix-reuse level swept in a routing A/B + router_policy: str = "" # round-robin | prefix-affinity | cache_aware | hybrid | adaptive | "" + prefix_reuse: float | None = None # prefix-reuse level swept in a routing A/B per_replica_cache_hit: float | None = None # per-replica prefix-cache hit (0..1) # Spec-decode acceptance length (the RESULT; the K lever is num_speculative_tokens). acceptance_length: float | None = None @@ -184,7 +185,7 @@ class AtlasCell: # Engine-init KV-cache token capacity (the nvfp4-kv "more KV tokens" headline). kv_cache_tokens: int | None = None # DeepEP / expert-parallel mode (sub-discriminator of parallel_strategy=EP). - ep_mode: str = "" # deepep-ll | deepep-ht | none | "" + ep_mode: str = "" # deepep-ll | deepep-ht | none | "" # Per-(cell,concurrency) DCGM utilization (0..1), promoted from campaign-level # dcgm_correlation.json so utilization-vs-concurrency is a flat atlas read. dcgm_sm_active: float | None = None @@ -205,24 +206,16 @@ class AtlasCell: def __post_init__(self) -> None: if self.status not in STATUSES: - raise ValueError( - f"AtlasCell.status must be one of {sorted(STATUSES)}, got {self.status!r}" - ) + raise ValueError(f"AtlasCell.status must be one of {sorted(STATUSES)}, got {self.status!r}") if self.parallel_strategy not in ("EP", "TP"): - raise ValueError( - f"AtlasCell.parallel_strategy must be 'EP' or 'TP', got {self.parallel_strategy!r}" - ) + raise ValueError(f"AtlasCell.parallel_strategy must be 'EP' or 'TP', got {self.parallel_strategy!r}") if self.backend and self.backend not in BACKENDS: - raise ValueError( - f"AtlasCell.backend must be one of {sorted(BACKENDS)} or empty, got {self.backend!r}" - ) + raise ValueError(f"AtlasCell.backend must be one of {sorted(BACKENDS)} or empty, got {self.backend!r}") # Derive serving_engine from backend when not explicitly provided. if not self.serving_engine: self.serving_engine = engine_for_backend(self.backend) if self.cache_mode not in ("warm", "cold", "unknown"): - raise ValueError( - f"AtlasCell.cache_mode must be 'warm', 'cold', or 'unknown', got {self.cache_mode!r}" - ) + raise ValueError(f"AtlasCell.cache_mode must be 'warm', 'cold', or 'unknown', got {self.cache_mode!r}") @property def has_metrics(self) -> bool: @@ -235,11 +228,7 @@ def has_latency_metrics(self) -> bool: or TTFT). A focus=latency run (e.g. c=1 decode / kernel probe) is a first-class published result even without request throughput, so the publish path counts these via ``plot_ready_latency_points``.""" - return ( - self.tpot_median_ms is not None - or self.itl_avg_ms is not None - or self.ttft_avg_ms is not None - ) + return self.tpot_median_ms is not None or self.itl_avg_ms is not None or self.ttft_avg_ms is not None @property def legend_key(self) -> tuple[str, str, int, str, bool]: @@ -257,10 +246,7 @@ def legend_key(self) -> tuple[str, str, int, str, bool]: def legend_label(self) -> str: """Human-readable label used in the matplotlib legend.""" mtp_suffix = " MTP" if self.mtp else "" - return ( - f"{self.hardware} {self.quant}{mtp_suffix} " - f"TP={self.tensor_parallel} {self.parallel_strategy}" - ) + return f"{self.hardware} {self.quant}{mtp_suffix} TP={self.tensor_parallel} {self.parallel_strategy}" def to_dict(self) -> dict[str, Any]: return asdict(self) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_aa_workload.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_aa_workload.py index fea3887..6c02295 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_aa_workload.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_aa_workload.py @@ -11,6 +11,7 @@ import importlib.util import json from pathlib import Path +from types import SimpleNamespace import pytest @@ -19,15 +20,33 @@ AA_SHAPES, MODE_DATASET_REPLAY, MODE_SYNTHETIC, + REDACTED_API_KEY, build_aiperf_command, generate_aa_dataset, normalize_outputs, + redact_aiperf_command, ) from tools.perf_tune_report.runners.common import CellConfig from tools.perf_tune_report.schema import BACKEND_AIPERF, STATUS_FAILED, STATUS_FULL # --- shapes --------------------------------------------------------------- +FAKE_API_KEY = "fake-aa-api-key-not-a-secret" + + +def _load_standalone_script(): + script = ( + Path(__file__).resolve().parents[3] + / "skills" + / "inference-aa-workload" + / "assets" + / "repro_artificialanalysis.py" + ) + spec = importlib.util.spec_from_file_location("aa_repro_script", script) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + def test_aa_shapes_match_methodology(): assert AA_SHAPES["aa-1k"].input_tokens == 1000 @@ -41,28 +60,13 @@ def test_aa_shapes_match_methodology(): def test_standalone_script_shapes_drift_guard(): """The bundled self-contained script must declare the same shapes as the package module (the script intentionally has no package import).""" - script = ( - Path(__file__).resolve().parents[3] - / "skills" - / "inference-aa-workload" - / "assets" - / "repro_artificialanalysis.py" - ) - spec = importlib.util.spec_from_file_location("aa_repro_script", script) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) + mod = _load_standalone_script() pkg = {k: (v.input_tokens, v.output_tokens) for k, v in AA_SHAPES.items()} assert mod.AA_SHAPES == pkg def _load_ttfo_probe(): - script = ( - Path(__file__).resolve().parents[3] - / "skills" - / "inference-aa-workload" - / "assets" - / "aa_ttfo_probe.py" - ) + script = Path(__file__).resolve().parents[3] / "skills" / "inference-aa-workload" / "assets" / "aa_ttfo_probe.py" spec = importlib.util.spec_from_file_location("aa_ttfo_probe", script) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) @@ -72,12 +76,8 @@ def _load_ttfo_probe(): def test_ttfo_probe_accepts_only_explicit_loopback_targets(): probe = _load_ttfo_probe() - assert probe.parse_probe_target( - "http://localhost:8000/v1/chat/completions" - ) == ("127.0.0.1", 8000) - assert probe.parse_probe_target( - "http://127.0.0.1:18000/v1/chat/completions" - ) == ("127.0.0.1", 18000) + assert probe.parse_probe_target("http://localhost:8000/v1/chat/completions") == ("127.0.0.1", 8000) + assert probe.parse_probe_target("http://127.0.0.1:18000/v1/chat/completions") == ("127.0.0.1", 18000) assert probe.parse_probe_target("http://[::1]:8000/v1/chat/completions") == ( "::1", 8000, @@ -161,7 +161,6 @@ def test_build_command_synthetic(): output_artifact_dir="/tmp/out", concurrency=4, request_count=20, - api_key="secret", tokenizer="m", mode=MODE_SYNTHETIC, ) @@ -171,11 +170,32 @@ def test_build_command_synthetic(): assert cmd[cmd.index("--output-tokens-mean") + 1] == "1500" assert "--extra-inputs" in cmd and "temperature:0" in cmd and "top_p:1" in cmd assert "min_tokens:1500" in cmd and "ignore_eos:true" in cmd - assert cmd[cmd.index("--api-key") + 1] == "secret" + assert "--api-key" not in cmd assert cmd[cmd.index("--concurrency") + 1] == "4" assert "--input-file" not in cmd +def test_redact_aiperf_command_copies_and_redacts_api_key_forms(): + split_form = ["aiperf", "profile", "--api-key", FAKE_API_KEY, "--model", "m"] + equals_form = ["aiperf", "profile", f"--api-key={FAKE_API_KEY}"] + + assert redact_aiperf_command(split_form) == [ + "aiperf", + "profile", + "--api-key", + REDACTED_API_KEY, + "--model", + "m", + ] + assert redact_aiperf_command(equals_form) == [ + "aiperf", + "profile", + f"--api-key={REDACTED_API_KEY}", + ] + assert split_form[3] == FAKE_API_KEY + assert equals_form[2].endswith(FAKE_API_KEY) + + def test_build_command_dataset_replay(): cmd = build_aiperf_command( AA_SHAPES["aa-1k"], @@ -268,9 +288,7 @@ def test_normalize_outputs_happy_path(tmp_path: Path): } ) ) - rows, status = normalize_outputs( - cell, cell_dir / "raw", cell_dir, shape=AA_SHAPES["aa-10k"], mode=MODE_SYNTHETIC - ) + rows, status = normalize_outputs(cell, cell_dir / "raw", cell_dir, shape=AA_SHAPES["aa-10k"], mode=MODE_SYNTHETIC) assert status == STATUS_FULL assert len(rows) == 1 r = rows[0] @@ -306,9 +324,7 @@ def test_normalize_outputs_aiperf_0_9_nested_schema(tmp_path: Path): } ) ) - rows, status = normalize_outputs( - cell, cell_dir / "raw", cell_dir, shape=AA_SHAPES["aa-1k"], mode=MODE_SYNTHETIC - ) + rows, status = normalize_outputs(cell, cell_dir / "raw", cell_dir, shape=AA_SHAPES["aa-1k"], mode=MODE_SYNTHETIC) assert status == STATUS_FULL assert len(rows) == 1 r = rows[0] @@ -322,9 +338,7 @@ def test_normalize_outputs_no_reports_is_failed(tmp_path: Path): cell = _cell() cell_dir = tmp_path / "cells" / cell.cell_id (cell_dir / "raw").mkdir(parents=True) - rows, status = normalize_outputs( - cell, cell_dir / "raw", cell_dir, shape=AA_SHAPES["aa-10k"], mode=MODE_SYNTHETIC - ) + rows, status = normalize_outputs(cell, cell_dir / "raw", cell_dir, shape=AA_SHAPES["aa-10k"], mode=MODE_SYNTHETIC) assert rows == [] assert status == STATUS_FAILED @@ -332,7 +346,7 @@ def test_normalize_outputs_no_reports_is_failed(tmp_path: Path): # --- runner dry-run ------------------------------------------------------- -def test_run_cell_aa_dry_run(tmp_path: Path): +def test_run_cell_aa_dry_run_redacts_api_key_from_receipts(tmp_path: Path): cell = CellConfig( cell_id="aa-1k", model="m", @@ -350,14 +364,115 @@ def test_run_cell_aa_dry_run(tmp_path: Path): shape_name="aa-1k", model="m", url="http://x:8000", + api_key=FAKE_API_KEY, mode=MODE_SYNTHETIC, dry_run=True, ) assert result.dry_run is True assert result.shape == "aa-1k" assert len(result.commands) == 2 # one per concurrency + assert FAKE_API_KEY not in json.dumps(result.commands) + assert all("--api-key" not in command for command in result.commands) # The .cmd capture file is written even on dry-run. - assert (result.cell_dir / "commands" / "aa-sweep.cmd").is_file() + receipt = result.cell_dir / "commands" / "aa-sweep.cmd" + assert receipt.is_file() + assert FAKE_API_KEY not in receipt.read_text() + assert "--api-key" not in receipt.read_text() + configs = sorted((result.cell_dir / "commands").glob("*.aiperf.json")) + assert len(configs) == 2 + assert all("${OPENAI_API_KEY}" in path.read_text() for path in configs) + assert all(FAKE_API_KEY not in path.read_text() for path in configs) + + +@pytest.mark.parametrize( + "aiperf_cmd", + [ + ["aiperf", "--api-key", FAKE_API_KEY], + ["aiperf", f"--api-key={FAKE_API_KEY}"], + ], + ids=["split", "equals"], +) +def test_run_cell_aa_rejects_api_key_in_custom_command(tmp_path: Path, aiperf_cmd: list[str]): + with pytest.raises(ValueError, match="managed API-key option"): + run_cell_aa( + _cell(), + tmp_path, + shape_name="aa-1k", + model="m", + url="http://x:8000", + api_key=FAKE_API_KEY, + aiperf_cmd=aiperf_cmd, + dry_run=True, + ) + + assert not (tmp_path / "cells").exists() + + +def test_run_cell_aa_passes_key_only_to_runtime_subprocess(tmp_path: Path): + calls = [] + + def runner(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=7, + stdout=f"provider output accidentally echoed {FAKE_API_KEY}", + stderr=f"provider error accidentally echoed {FAKE_API_KEY}", + ) + + result = run_cell_aa( + _cell(), + tmp_path, + shape_name="aa-1k", + model="m", + url="http://x:8000", + api_key=FAKE_API_KEY, + mode=MODE_SYNTHETIC, + subprocess_runner=runner, + ) + + assert len(calls) == 1 + command, kwargs = calls[0] + assert FAKE_API_KEY not in command + assert "--api-key" not in command + assert "--config" in command + assert kwargs["env"]["OPENAI_API_KEY"] == FAKE_API_KEY + assert FAKE_API_KEY not in json.dumps(result.commands) + assert all("--api-key" not in item for item in result.commands) + assert FAKE_API_KEY not in (result.cell_dir / "commands" / "aa-sweep.cmd").read_text() + assert FAKE_API_KEY not in (result.cell_dir / "commands" / "aa-sweep.stdout").read_text() + assert FAKE_API_KEY not in (result.cell_dir / "commands" / "aa-sweep.stderr").read_text() + assert REDACTED_API_KEY in (result.cell_dir / "commands" / "aa-sweep.stdout").read_text() + assert REDACTED_API_KEY in (result.cell_dir / "commands" / "aa-sweep.stderr").read_text() + + +def test_run_cell_aa_kube_streams_key_without_argv_exposure(tmp_path: Path): + calls = [] + + def runner(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + result = run_cell_aa( + _cell(), + tmp_path, + shape_name="aa-1k", + model="m", + url="http://x:8000", + api_key=FAKE_API_KEY, + mode=MODE_SYNTHETIC, + kube={"namespace": "ns", "bench_pod": "bench", "kube_context": "ctx"}, + spec_scrape=False, + subprocess_runner=runner, + ) + + bench_command, bench_kwargs = calls[0] + assert bench_command[:3] == ["kubectl", "exec", "-i"] + assert FAKE_API_KEY not in bench_command + assert "--api-key" not in bench_command + assert bench_kwargs["input"].startswith(f"{FAKE_API_KEY}\n") + assert "${OPENAI_API_KEY}" in bench_kwargs["input"] + assert FAKE_API_KEY not in bench_kwargs["input"].split("\n", 1)[1] + assert FAKE_API_KEY not in json.dumps(result.commands) def test_run_cell_aa_unknown_shape(tmp_path: Path): @@ -378,7 +493,7 @@ def test_run_cell_aa_unknown_shape(tmp_path: Path): # --- CLI dispatch --------------------------------------------------------- -def test_cli_cell_run_aa_dry_run(tmp_path: Path, capsys): +def test_cli_cell_run_aa_dry_run_redacts_api_key(tmp_path: Path, capsys, monkeypatch): from tools.perf_tune_report.perf_tune_report_cli import build_parser campaign = tmp_path / "20260529T000000Z-aa" @@ -408,6 +523,7 @@ def test_cli_cell_run_aa_dry_run(tmp_path: Path, capsys): import yaml (campaign / "config.yaml").write_text(yaml.safe_dump(config)) + monkeypatch.setenv("WANDB_INFERENCE_API_KEY", FAKE_API_KEY) parser = build_parser() args = parser.parse_args( @@ -425,8 +541,107 @@ def test_cli_cell_run_aa_dry_run(tmp_path: Path, capsys): ) rc = args.func(args) assert rc == 0 - out = json.loads(capsys.readouterr().out) + stdout = capsys.readouterr().out + assert FAKE_API_KEY not in stdout + out = json.loads(stdout) assert out["backend"] == "aa" assert out["aa_shape"] == "aa-10k" assert out["dry_run"] is True assert len(out["commands"]) == 2 + assert "--api-key" not in json.dumps(out["commands"]) + receipt = campaign / "cells" / "aa-10k" / "commands" / "aa-sweep.cmd" + assert FAKE_API_KEY not in receipt.read_text() + + +def test_standalone_dry_run_omits_managed_api_key(tmp_path: Path, monkeypatch, capsys): + mod = _load_standalone_script() + monkeypatch.chdir(tmp_path) + + rc = mod.main( + [ + "--aiperf-bin", + "aiperf", + "--api-key", + FAKE_API_KEY, + "--shapes", + "aa-1k", + "--dry-run", + ] + ) + + assert rc == 0 + stdout = capsys.readouterr().out + assert FAKE_API_KEY not in stdout + assert "--api-key" not in stdout + assert not (tmp_path / "artifacts").exists() + + +@pytest.mark.parametrize( + "aiperf_bin", + [ + f"aiperf --api-key {FAKE_API_KEY}", + f"aiperf --api-key={FAKE_API_KEY}", + ], + ids=["split", "equals"], +) +def test_standalone_rejects_api_key_in_custom_command(aiperf_bin: str, capsys): + mod = _load_standalone_script() + + rc = mod.main( + [ + "--aiperf-bin", + aiperf_bin, + "--api-key", + FAKE_API_KEY, + "--shapes", + "aa-1k", + "--dry-run", + ] + ) + + captured = capsys.readouterr() + assert rc == 2 + assert "must not include --api-key" in captured.err + assert FAKE_API_KEY not in captured.out + assert FAKE_API_KEY not in captured.err + + +def test_standalone_runtime_passes_api_key_to_subprocess(tmp_path: Path, monkeypatch, capsys): + mod = _load_standalone_script() + calls = [] + + def runner(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace( + returncode=0, + stdout=f"child stdout echoed {FAKE_API_KEY}\n", + stderr=f"child stderr echoed {FAKE_API_KEY}\n", + ) + + monkeypatch.setattr(mod.subprocess, "run", runner) + rc = mod.main( + [ + "--aiperf-bin", + "aiperf", + "--api-key", + FAKE_API_KEY, + "--artifact-root", + str(tmp_path / "artifacts"), + "--shapes", + "aa-1k", + ] + ) + + assert rc == 0 + command, kwargs = calls[0] + assert FAKE_API_KEY not in command + assert "--api-key" not in command + assert "--config" in command + assert kwargs["env"]["OPENAI_API_KEY"] == FAKE_API_KEY + assert kwargs["capture_output"] is True + assert kwargs["text"] is True + captured = capsys.readouterr() + assert FAKE_API_KEY not in captured.out + assert FAKE_API_KEY not in captured.err + assert REDACTED_API_KEY in captured.out + assert REDACTED_API_KEY in captured.err diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_campaign_run.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_campaign_run.py index 67aa636..8e1e00d 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_campaign_run.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_campaign_run.py @@ -3,13 +3,10 @@ from __future__ import annotations import json -from pathlib import Path -from typing import Sequence import pytest from tools.perf_tune_report.orchestrator.campaign_run import ( - CampaignRunResult, CellPlan, CellStepResult, StepFns, @@ -18,7 +15,6 @@ run_one_cell, ) - # ----------------------------------------------------------------------------- # CellPlan schema # ----------------------------------------------------------------------------- @@ -69,11 +65,21 @@ def test_cell_plan_rejects_empty_concurrencies(): def _mk_cell_result(cell_id: str, verdict: str, bench_ok: bool = True) -> CellStepResult: return CellStepResult( - cell_id=cell_id, started_at="t", ended_at="t", elapsed_s=0.0, - drain_ok=True, helm_ok=True, warmup_ok=True, bench_ok=bench_ok, - zymtrace_ok=True, import_ok=True, aggregate_ok=True, - render_ok=True, baseline_record_ok=True, - baseline_diff_verdict=verdict, resume_ok=True, + cell_id=cell_id, + started_at="t", + ended_at="t", + elapsed_s=0.0, + drain_ok=True, + helm_ok=True, + warmup_ok=True, + bench_ok=bench_ok, + zymtrace_ok=True, + import_ok=True, + aggregate_ok=True, + render_ok=True, + baseline_record_ok=True, + baseline_diff_verdict=verdict, + resume_ok=True, ) @@ -114,6 +120,7 @@ def _make_stub_fns( resume_raises: bool = False, ) -> StepFns: """Build a StepFns where every step returns the configured outcome.""" + def _resume_fn(nodes, cell_id): if resume_raises: raise RuntimeError("simulated resume failure") @@ -171,9 +178,9 @@ def test_run_one_cell_always_resumes_on_bench_failure(tmp_path): comparator_baseline="", step_fns=fns, ) - assert result.drain_ok # drain succeeded - assert not result.bench_ok # bench failed - assert result.resume_ok # CRITICAL: resume still ran in finally + assert result.drain_ok # drain succeeded + assert not result.bench_ok # bench failed + assert result.resume_ok # CRITICAL: resume still ran in finally def test_run_one_cell_handles_resume_exception(tmp_path): @@ -293,9 +300,9 @@ def test_run_one_cell_aa_skips_helm_and_warmup(tmp_path): comparator_baseline="", step_fns=fns, ) - assert helm_calls == [] # helm_upgrade never invoked + assert helm_calls == [] # helm_upgrade never invoked assert warmup_calls == [] # warmup never invoked - assert result.helm_ok # recorded as no-op pass + assert result.helm_ok # recorded as no-op pass assert result.warmup_ok assert result.bench_ok @@ -307,9 +314,12 @@ def test_run_one_cell_aa_skips_helm_and_warmup(tmp_path): def test_step_bench_aa_returns_false_when_config_incomplete(tmp_path): from tools.perf_tune_report.orchestrator.production_steps import step_bench + # missing url + shape cell = CellPlan( - id="aa-1k", backend="aa", concurrencies=(1,), + id="aa-1k", + backend="aa", + concurrencies=(1,), backend_config={"model": "m"}, ) assert step_bench(tmp_path, cell) is False @@ -317,8 +327,12 @@ def test_step_bench_aa_returns_false_when_config_incomplete(tmp_path): def test_step_bench_aiperf_returns_false_when_config_incomplete(tmp_path): from tools.perf_tune_report.orchestrator.production_steps import step_bench + cell = CellPlan( - id="replay", backend="aiperf", concurrencies=(1,), backend_config={}, + id="replay", + backend="aiperf", + concurrencies=(1,), + backend_config={}, ) assert step_bench(tmp_path, cell) is False @@ -426,8 +440,10 @@ def test_run_campaign_fail_fast_on_red(tmp_path): ] # Build a custom stub that returns RED only for cell-2-bad. diffs = {"cell-1": "GREEN", "cell-2-bad": "RED", "cell-3": "GREEN"} + def _baseline_diff(cdir, cell_id, comp): return diffs.get(cell_id, "NA") + fns = StepFns( drain=lambda nodes, cell_id: True, helm_upgrade=lambda ns, rel, chart, overrides: True, @@ -495,7 +511,7 @@ def test_run_campaign_continue_on_red(tmp_path): def test_run_campaign_writes_receipts(tmp_path): cells = [CellPlan(id="cell-1", backend="vllm-sweep", concurrencies=(1,))] fns = _make_stub_fns() - result = run_campaign( + run_campaign( cells, campaign_dir=tmp_path, target_release="rel", @@ -519,32 +535,58 @@ def test_contract_verb_set(): """The perf_tune_report CONTRACT verb set, including champion_select and import_variant_ab.""" from tools.perf_tune_report.perf_tune_report_cli import CONTRACT + assert set(CONTRACT) == { - "campaign_init", "cell_run", "atlas_aggregate", - "report_render", "report_smoke", "publish_to_lake", - "import_perf_bench", "campaign_run", - "kernel_profile", "graph_diff", - "raw_bench_compare", "import_ncu", "import_nsys", "dcgm_correlate", - "experiments_index", "tpm_summary", "value_view", - "import_roofline_sweep", "fleet_leaderboard", - "champion_select", "import_variant_ab", - "capture_plan", "materialize_capture_reuse", - "experiment_inventory", "portability_view", "import_model_eval", "trend_view", - "kernel_reproducer_scaffold", "import_workloads", + "campaign_init", + "cell_run", + "atlas_aggregate", + "report_render", + "report_smoke", + "publish_to_lake", + "import_perf_bench", + "campaign_run", + "kernel_profile", + "graph_diff", + "raw_bench_compare", + "import_ncu", + "import_nsys", + "dcgm_correlate", + "experiments_index", + "tpm_summary", + "value_view", + "import_roofline_sweep", + "fleet_leaderboard", + "champion_select", + "import_variant_ab", + "capture_plan", + "materialize_capture_reuse", + "experiment_inventory", + "portability_view", + "import_model_eval", + "trend_view", + "kernel_reproducer_scaffold", + "import_workloads", } def test_contract_campaign_run_is_ack_gated(): from tools.perf_tune_report.perf_tune_report_cli import CONTRACT, build_parser + assert CONTRACT["campaign_run"]["safety"] == "mutates_cluster" assert CONTRACT["campaign_run"]["ack"] == "--i-understand-this-mutates-cluster" assert CONTRACT["campaign_run"]["ack_exempt_when"] == ("--dry-run",) assert "--config" in CONTRACT["campaign_run"]["required"] assert "--campaign" in CONTRACT["campaign_run"]["required"] - parsed = build_parser().parse_args([ - "campaign_run", "--config", "matrix.yaml", "--campaign", "campaign", - "--i-understand-this-mutates-cluster", - ]) + parsed = build_parser().parse_args( + [ + "campaign_run", + "--config", + "matrix.yaml", + "--campaign", + "campaign", + "--i-understand-this-mutates-cluster", + ] + ) assert parsed.i_understand_this_mutates_cluster is True assert not hasattr(parsed, "i_understand_this_submits_jobs") diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_capture_signature.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_capture_signature.py index 2b8b971..c00d253 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_capture_signature.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_capture_signature.py @@ -57,15 +57,14 @@ def test_signature_distinguishes_first_class_variant_fields(): """The 2026-06-07 first-class variant fields change the signature (so e.g. MTP-K=2 vs K=3, async on vs off are distinct variants), and the legacy extra fallback holds.""" base = _row("base") - assert signature_for_row(base).hash != signature_for_row( - _row("k3", num_speculative_tokens=3)).hash - assert signature_for_row(base).hash != signature_for_row( - _row("async", async_scheduling=True)).hash - assert signature_for_row(base).hash != signature_for_row( - _row("pc", enable_prefix_caching=True)).hash + assert signature_for_row(base).hash != signature_for_row(_row("k3", num_speculative_tokens=3)).hash + assert signature_for_row(base).hash != signature_for_row(_row("async", async_scheduling=True)).hash + assert signature_for_row(base).hash != signature_for_row(_row("pc", enable_prefix_caching=True)).hash # first-class max_num_seqs and the legacy extra["max_num_seqs"] resolve to the same value - assert signature_for_row(_row("a", max_num_seqs=96, extra={})).hash == \ - signature_for_row(_row("b", extra={"max_num_seqs": 96})).hash + assert ( + signature_for_row(_row("a", max_num_seqs=96, extra={})).hash + == signature_for_row(_row("b", extra={"max_num_seqs": 96})).hash + ) # None max_num_batched_tokens (accuracy row) does not crash the signature assert signature_for_row(_row("acc", max_num_batched_tokens=None)).hash @@ -73,26 +72,28 @@ def test_signature_distinguishes_first_class_variant_fields(): def test_signature_changes_on_exact_variant_axes(): base = _row("base") assert signature_for_row(base).hash == signature_for_row(_row("same")).hash - assert signature_for_row(base).hash != signature_for_row( - _row("tp1", tensor_parallel=1, parallel_strategy="TP") - ).hash - assert signature_for_row(base).hash != signature_for_row( - _row("mns128", extra={"max_num_seqs": 128}) - ).hash - assert signature_for_row(base).hash != signature_for_row( - _row("sglang", backend="sglang-sweep") - ).hash - assert signature_for_row(base).hash != signature_for_row( - _row("fimoe", extra={"max_num_seqs": 96, "env": {"VLLM_USE_FLASHINFER_MOE_FP4": "1"}}) - ).hash + assert ( + signature_for_row(base).hash != signature_for_row(_row("tp1", tensor_parallel=1, parallel_strategy="TP")).hash + ) + assert signature_for_row(base).hash != signature_for_row(_row("mns128", extra={"max_num_seqs": 128})).hash + assert signature_for_row(base).hash != signature_for_row(_row("sglang", backend="sglang-sweep")).hash + assert ( + signature_for_row(base).hash + != signature_for_row( + _row("fimoe", extra={"max_num_seqs": 96, "env": {"VLLM_USE_FLASHINFER_MOE_FP4": "1"}}) + ).hash + ) def test_capture_plan_groups_missing_and_finds_exact_reuse(tmp_path: Path): source = _write_campaign(tmp_path / "source", [_row("src")]) - target = _write_campaign(tmp_path / "target", [ - _row("match"), - _row("different-mns", extra={"max_num_seqs": 128}), - ]) + target = _write_campaign( + tmp_path / "target", + [ + _row("match"), + _row("different-mns", extra={"max_num_seqs": 128}), + ], + ) src_cell = source / "cells" / "src" src_cell.mkdir(parents=True) (src_cell / "roofline_sweep.json").write_text('{"schema":"roofline_sweep_points_v1"}\n') @@ -104,19 +105,19 @@ def test_capture_plan_groups_missing_and_finds_exact_reuse(tmp_path: Path): assert cand.source_cell == "src" assert cand.target_cell == "match" - missing_roofline = [ - g for g in plan.missing_groups - if g.artifact == "roofline_sweep.json" - ] + missing_roofline = [g for g in plan.missing_groups if g.artifact == "roofline_sweep.json"] assert len(missing_roofline) == 1 assert missing_roofline[0].cells[0]["cell_id"] == "different-mns" def test_capture_plan_resolves_legacy_arm_artifact_dir(tmp_path: Path): """Legacy variant imports can have atlas cell_id != cells/.""" - camp = _write_campaign(tmp_path / "campaign", [ - _row("qwen3-next-xeng-vd-v021-Kengine", extra={"arm": "qwen3-next-xeng-vd-v021"}), - ]) + camp = _write_campaign( + tmp_path / "campaign", + [ + _row("qwen3-next-xeng-vd-v021-Kengine", extra={"arm": "qwen3-next-xeng-vd-v021"}), + ], + ) physical = camp / "cells" / "qwen3-next-xeng-vd-v021" physical.mkdir(parents=True) (physical / "dcgm_correlation.json").write_text('{"schema_version":1}\n') @@ -155,13 +156,18 @@ def test_capture_plan_and_materialize_cli(tmp_path: Path): (src_cell / "dcgm_correlation.json").write_text('{"schema_version":1}\n') plan_path = tmp_path / "plan.json" - rc = main([ - "capture_plan", - "--campaign", str(target), - "--source-campaign", str(source), - "--out", str(plan_path), - "--json", - ]) + rc = main( + [ + "capture_plan", + "--campaign", + str(target), + "--source-campaign", + str(source), + "--out", + str(plan_path), + "--json", + ] + ) assert rc == 0 plan = json.loads(plan_path.read_text()) assert plan["schema_version"] == "capture_plan_v1" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_champion_select.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_champion_select.py index e0c9b72..e11e149 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_champion_select.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_champion_select.py @@ -11,7 +11,7 @@ from __future__ import annotations import json -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path import pytest @@ -24,11 +24,20 @@ # --------------------------------------------------------------------------- # def _row(cell, eng, c, tput_gpu, tpot): return { - "cell_id": cell, "model": "org/Model-X", "hardware": "GB300", "quant": "NVFP4", - "tensor_parallel": 4, "parallel_strategy": "TP", "mtp": False, - "max_num_batched_tokens": 12288, "concurrency": c, "status": "full", - "ttft_avg_ms": 120.0, "request_throughput_avg": 5.0, - "output_tps_per_gpu": tput_gpu, "tpot_median_ms": tpot, + "cell_id": cell, + "model": "org/Model-X", + "hardware": "GB300", + "quant": "NVFP4", + "tensor_parallel": 4, + "parallel_strategy": "TP", + "mtp": False, + "max_num_batched_tokens": 12288, + "concurrency": c, + "status": "full", + "ttft_avg_ms": 120.0, + "request_throughput_avg": 5.0, + "output_tps_per_gpu": tput_gpu, + "tpot_median_ms": tpot, "backend": "sglang-sweep" if eng == "sglang" else "vllm-sweep", } @@ -37,42 +46,71 @@ def _stage_campaign(tmp_path: Path, *, with_sol=True) -> Path: camp = tmp_path / "20260607T000000Z-model-x-crossengine" (camp / "cells").mkdir(parents=True) rows = [ - _row("mx-v-base", "vllm", 32, 300.0, 40.0), # baseline (vLLM) - _row("mx-v-ep", "vllm", 32, 360.0, 42.0), # +20% tput, TPOT 42 <= SLO 44 -> PASS - _row("mx-v-cg", "vllm", 32, 330.0, 55.0), # +10% tput but TPOT 55 > SLO -> SLO-FAIL + _row("mx-v-base", "vllm", 32, 300.0, 40.0), # baseline (vLLM) + _row("mx-v-ep", "vllm", 32, 360.0, 42.0), # +20% tput, TPOT 42 <= SLO 44 -> PASS + _row("mx-v-cg", "vllm", 32, 330.0, 55.0), # +10% tput but TPOT 55 > SLO -> SLO-FAIL _row("mx-s-base", "sglang", 32, 280.0, 35.0), # sglang baseline _row("mx-s-attn", "sglang", 32, 410.0, 30.0), # cross-engine champion ] - (camp / "atlas.jsonl").write_text( - "\n".join(json.dumps(r, sort_keys=True) for r in rows) + "\n" - ) + (camp / "atlas.jsonl").write_text("\n".join(json.dumps(r, sort_keys=True) for r in rows) + "\n") if with_sol: # 4-layer SoL artifacts for the cross-engine champion (mx-s-attn). champ = camp / "cells" / "mx-s-attn" champ.mkdir(parents=True) - (champ / "kernels.json").write_text(json.dumps({ - "schema_version": 1, "captured_sources": ["zymtrace"], - "top_kernels": [], "per_gpu": [], "top_python_during_cuda": [], - "per_category": {"MoE": 5000, "FMHA": 3000, "NCCL": 1000}, - })) - (champ / "dcgm_correlation.json").write_text(json.dumps({ - "schema_version": 1, "captured_sources": ["dcgm"], "hw_key": "gb300_nvl72", "queries": [], - "resources": [ - {"peak_key": "hbm3e_tbps", "metric": "DCGM_FI_PROF_DRAM_ACTIVE", "sol_pct": 24.2}, - {"peak_key": "nvfp4_dense_pflops", "metric": "PIPE_TENSOR_ACTIVE", "sol_pct": 3.1}, - ], - "per_category_attribution": [{"category": "MoE", "time_share_pct": 55.0, "sol_pct_bw": 0.2}], - })) - (champ / "ncu_kernels.json").write_text(json.dumps({ - "schema_version": 1, "captured_sources": ["ncu"], "hw_key": "gb300_nvl72", - "kernels": [{"name": "k0", "sol_pct": 80}], - })) - (champ / "roofline_sweep.json").write_text(json.dumps({ - "schema": "roofline_sweep_points_v1", "hardware": "GB300", "tensor_parallel": 4, - "quant": "NVFP4", "model": "org/Model-X", - "decode": [{"phase": "decode", "c": 32, "tensor_active": 0.031, "dram_active": 0.242, "sm_active": 0.40}], - "prefill": [{"phase": "prefill", "isl": 4096, "tensor_active": 0.6, "dram_active": 0.3, "sm_active": 0.8}], - })) + (champ / "kernels.json").write_text( + json.dumps( + { + "schema_version": 1, + "captured_sources": ["zymtrace"], + "top_kernels": [], + "per_gpu": [], + "top_python_during_cuda": [], + "per_category": {"MoE": 5000, "FMHA": 3000, "NCCL": 1000}, + } + ) + ) + (champ / "dcgm_correlation.json").write_text( + json.dumps( + { + "schema_version": 1, + "captured_sources": ["dcgm"], + "hw_key": "gb300_nvl72", + "queries": [], + "resources": [ + {"peak_key": "hbm3e_tbps", "metric": "DCGM_FI_PROF_DRAM_ACTIVE", "sol_pct": 24.2}, + {"peak_key": "nvfp4_dense_pflops", "metric": "PIPE_TENSOR_ACTIVE", "sol_pct": 3.1}, + ], + "per_category_attribution": [{"category": "MoE", "time_share_pct": 55.0, "sol_pct_bw": 0.2}], + } + ) + ) + (champ / "ncu_kernels.json").write_text( + json.dumps( + { + "schema_version": 1, + "captured_sources": ["ncu"], + "hw_key": "gb300_nvl72", + "kernels": [{"name": "k0", "sol_pct": 80}], + } + ) + ) + (champ / "roofline_sweep.json").write_text( + json.dumps( + { + "schema": "roofline_sweep_points_v1", + "hardware": "GB300", + "tensor_parallel": 4, + "quant": "NVFP4", + "model": "org/Model-X", + "decode": [ + {"phase": "decode", "c": 32, "tensor_active": 0.031, "dram_active": 0.242, "sm_active": 0.40} + ], + "prefill": [ + {"phase": "prefill", "isl": 4096, "tensor_active": 0.6, "dram_active": 0.3, "sm_active": 0.8} + ], + } + ) + ) return camp @@ -85,15 +123,27 @@ def test_accuracy_floor_derives_gate_from_local_eval(tmp_path: Path): camp = _stage_campaign(tmp_path) atlas = camp / "atlas.jsonl" eval_row = { - "cell_id": "model-eval", "model": "org/Model-X", "hardware": "GB300", "quant": "NVFP4", - "tensor_parallel": 4, "parallel_strategy": "TP", "mtp": False, - "max_num_batched_tokens": 0, "concurrency": 1, "status": "full", - "extra": {"metric_kind": "eval_acc", - "quality_metrics": {"gpqa_acc": 0.52, "mmlu_pro_acc": 0.90}}, + "cell_id": "model-eval", + "model": "org/Model-X", + "hardware": "GB300", + "quant": "NVFP4", + "tensor_parallel": 4, + "parallel_strategy": "TP", + "mtp": False, + "max_num_batched_tokens": 0, + "concurrency": 1, + "status": "full", + "extra": {"metric_kind": "eval_acc", "quality_metrics": {"gpqa_acc": 0.52, "mmlu_pro_acc": 0.90}}, } atlas.write_text(atlas.read_text() + json.dumps(eval_row, sort_keys=True) + "\n") - common = dict(focus="throughput", focus_c=32, top=3, trials=3, same_node=True, - workloads_present=("aa", "sonnet", "sharegpt", "random", "code")) + common = dict( + focus="throughput", + focus_c=32, + top=3, + trials=3, + same_node=True, + workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), + ) # floor below the worst measured metric (0.52) -> pass res = cs.select(camp, accuracy_floor=0.40, **common) acc = next(g for g in res.gates if g.name == "accuracy") @@ -108,12 +158,18 @@ def test_accuracy_floor_derives_gate_from_local_eval(tmp_path: Path): def test_select_picks_cross_engine_champion(tmp_path: Path): camp = _stage_campaign(tmp_path) - res = cs.select(camp, focus="throughput", focus_c=32, top=3, - trials=3, same_node=True, - workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), - accuracy_gate="pass") + res = cs.select( + camp, + focus="throughput", + focus_c=32, + top=3, + trials=3, + same_node=True, + workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), + accuracy_gate="pass", + ) assert res.baseline_cell == "mx-v-base" - assert res.recommended_cell == "mx-s-attn" # cross-engine winner + assert res.recommended_cell == "mx-s-attn" # cross-engine winner assert res.recommended_engine == "sglang" champ = next(v for v in res.variants if v.cell_id == "mx-s-attn") assert champ.pct_win_vs_baseline == pytest.approx(36.7, abs=0.2) @@ -147,12 +203,21 @@ def test_select_resolves_legacy_arm_artifact_dir(tmp_path: Path): (camp / "atlas.jsonl").write_text(json.dumps(row, sort_keys=True) + "\n") physical = camp / "cells" / "mx-s-attn" physical.mkdir(parents=True) - (physical / "roofline_sweep.json").write_text(json.dumps({ - "schema": "roofline_sweep_points_v1", "hardware": "GB300", "tensor_parallel": 4, - "quant": "NVFP4", "model": "org/Model-X", - "decode": [{"phase": "decode", "c": 32, "tensor_active": 0.031, "dram_active": 0.242, "sm_active": 0.40}], - "prefill": [], - })) + (physical / "roofline_sweep.json").write_text( + json.dumps( + { + "schema": "roofline_sweep_points_v1", + "hardware": "GB300", + "tensor_parallel": 4, + "quant": "NVFP4", + "model": "org/Model-X", + "decode": [ + {"phase": "decode", "c": 32, "tensor_active": 0.031, "dram_active": 0.242, "sm_active": 0.40} + ], + "prefill": [], + } + ) + ) res = cs.select(camp, focus="throughput", focus_c=32, top=1) variant = res.variants[0] @@ -167,20 +232,32 @@ def test_select_resolves_legacy_arm_artifact_dir(tmp_path: Path): # --------------------------------------------------------------------------- # def test_verdict_when_all_gates_pass(tmp_path: Path): camp = _stage_campaign(tmp_path) - res = cs.select(camp, focus="throughput", focus_c=32, top=3, - trials=3, same_node=True, - workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), - accuracy_gate="pass") + res = cs.select( + camp, + focus="throughput", + focus_c=32, + top=3, + trials=3, + same_node=True, + workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), + accuracy_gate="pass", + ) assert res.tier == "verdict" assert all(g.status == "pass" for g in res.gates) def test_draft_when_workloads_missing(tmp_path: Path): camp = _stage_campaign(tmp_path) - res = cs.select(camp, focus="throughput", focus_c=32, top=3, - trials=3, same_node=True, - workloads_present=("aa", "sonnet"), # missing sharegpt/random/code - accuracy_gate="pass") + res = cs.select( + camp, + focus="throughput", + focus_c=32, + top=3, + trials=3, + same_node=True, + workloads_present=("aa", "sonnet"), # missing sharegpt/random/code + accuracy_gate="pass", + ) assert res.tier == "draft" mw = next(g for g in res.gates if g.name == "multi_workload") assert mw.status == "fail" @@ -188,18 +265,30 @@ def test_draft_when_workloads_missing(tmp_path: Path): def test_draft_when_accuracy_unknown(tmp_path: Path): camp = _stage_campaign(tmp_path) - res = cs.select(camp, focus="throughput", focus_c=32, top=3, - trials=3, same_node=True, - workloads_present=("aa", "sonnet", "sharegpt", "random", "code")) + res = cs.select( + camp, + focus="throughput", + focus_c=32, + top=3, + trials=3, + same_node=True, + workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), + ) assert res.tier == "draft" # accuracy_gate defaults unknown def test_draft_when_champion_not_byte_grounded(tmp_path: Path): camp = _stage_campaign(tmp_path, with_sol=False) # no dcgm/roofline -> no L3 - res = cs.select(camp, focus="throughput", focus_c=32, top=3, - trials=3, same_node=True, - workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), - accuracy_gate="pass") + res = cs.select( + camp, + focus="throughput", + focus_c=32, + top=3, + trials=3, + same_node=True, + workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), + accuracy_gate="pass", + ) assert res.tier == "draft" dg = next(g for g in res.gates if g.name == "dcgm_grounded") assert dg.status == "fail" @@ -234,16 +323,22 @@ def test_select_missing_atlas_raises(tmp_path: Path): # --------------------------------------------------------------------------- # def test_lake_champion_table_and_campaign_fields(tmp_path: Path): pytest.importorskip("pyarrow") - from tools.perf_tune_report.lake_writer import build_champion_table, build_campaign_row + from tools.perf_tune_report.lake_writer import build_campaign_row, build_champion_table from tools.perf_tune_report.schema import read_jsonl camp = _stage_campaign(tmp_path) - res = cs.select(camp, focus="throughput", focus_c=32, top=3, - trials=3, same_node=True, - workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), - accuracy_gate="pass") + res = cs.select( + camp, + focus="throughput", + focus_c=32, + top=3, + trials=3, + same_node=True, + workloads_present=("aa", "sonnet", "sharegpt", "random", "code"), + accuracy_gate="pass", + ) cs.write_outputs(res, camp) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) ct = build_champion_table(camp, camp.name, captured_at_utc=now, published_at_utc=now) recs = {r["cell_id"]: r for r in ct.to_pylist()} @@ -266,7 +361,7 @@ def test_lake_champion_table_empty_without_champion_json(tmp_path: Path): from tools.perf_tune_report.lake_writer import build_champion_table camp = _stage_campaign(tmp_path) # no champion_select.json written - now = datetime.now(timezone.utc) + now = datetime.now(UTC) ct = build_champion_table(camp, camp.name, captured_at_utc=now, published_at_utc=now) assert ct.num_rows == 0 diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_correlate_from_frozen.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_correlate_from_frozen.py index 2868b9c..83e0ddb 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_correlate_from_frozen.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_correlate_from_frozen.py @@ -11,7 +11,6 @@ import json import sys -import textwrap from pathlib import Path import pytest @@ -27,7 +26,6 @@ write_correlation, ) - _MIN_CEILINGS: dict = { "b200_sm100": { "hw_name": "NVIDIA B200 (test)", diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_datacapture_gaps.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_datacapture_gaps.py index d1762e8..2f30c7c 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_datacapture_gaps.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_datacapture_gaps.py @@ -9,25 +9,45 @@ from __future__ import annotations from tools.perf_tune_report.lake_writer import build_atlas_table -from tools.perf_tune_report.schema import AtlasCell, STATUS_FULL, read_jsonl, write_jsonl +from tools.perf_tune_report.schema import STATUS_FULL, AtlasCell, read_jsonl, write_jsonl def _cell_with_gaps() -> AtlasCell: return AtlasCell( - cell_id="kvr-reuse10", model="GLM-5.1", hardware="GB300", quant="NVFP4", - tensor_parallel=4, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=8192, concurrency=64, status=STATUS_FULL, - router_policy="prefix-affinity", prefix_reuse=10.0, per_replica_cache_hit=0.83, - acceptance_length=1.955, kv_cache_tokens=524288, ep_mode="deepep-ll", - dcgm_sm_active=0.41, dcgm_dram_active=0.28, dcgm_tensor_active=0.12, + cell_id="kvr-reuse10", + model="GLM-5.1", + hardware="GB300", + quant="NVFP4", + tensor_parallel=4, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=8192, + concurrency=64, + status=STATUS_FULL, + router_policy="prefix-affinity", + prefix_reuse=10.0, + per_replica_cache_hit=0.83, + acceptance_length=1.955, + kv_cache_tokens=524288, + ep_mode="deepep-ll", + dcgm_sm_active=0.41, + dcgm_dram_active=0.28, + dcgm_tensor_active=0.12, ) def test_datacapture_gap_fields_default_safely(): c = AtlasCell( - cell_id="x", model="m", hardware="GB300", quant="NVFP4", - tensor_parallel=4, parallel_strategy="TP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status=STATUS_FULL, + cell_id="x", + model="m", + hardware="GB300", + quant="NVFP4", + tensor_parallel=4, + parallel_strategy="TP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status=STATUS_FULL, ) assert c.router_policy == "" and c.ep_mode == "" assert c.prefix_reuse is None and c.per_replica_cache_hit is None @@ -54,8 +74,15 @@ def test_datacapture_gap_fields_in_atlas_table(): table = build_atlas_table([_cell_with_gaps()], campaign_id="kvr-20260607T000000Z") names = set(table.schema.names) for col in ( - "router_policy", "prefix_reuse", "per_replica_cache_hit", "acceptance_length", - "kv_cache_tokens", "ep_mode", "dcgm_sm_active", "dcgm_dram_active", "dcgm_tensor_active", + "router_policy", + "prefix_reuse", + "per_replica_cache_hit", + "acceptance_length", + "kv_cache_tokens", + "ep_mode", + "dcgm_sm_active", + "dcgm_dram_active", + "dcgm_tensor_active", ): assert col in names, f"{col} missing from atlas_v1 schema" d = table.to_pydict() @@ -69,9 +96,16 @@ def test_datacapture_gap_fields_in_atlas_table(): def test_datacapture_gap_minimal_cell_atlas_table_defaults(): # A cell that does not set the gap fields still builds a valid table (defaults). minimal = AtlasCell( - cell_id="x", model="m", hardware="GB300", quant="NVFP4", - tensor_parallel=4, parallel_strategy="TP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status=STATUS_FULL, + cell_id="x", + model="m", + hardware="GB300", + quant="NVFP4", + tensor_parallel=4, + parallel_strategy="TP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status=STATUS_FULL, ) d = build_atlas_table([minimal], campaign_id="c-20260607T000000Z").to_pydict() assert d["router_policy"][0] == "" and d["ep_mode"][0] == "" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_correlate.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_correlate.py index 38ad326..d63c81b 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_correlate.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_correlate.py @@ -18,7 +18,7 @@ import json import sys from dataclasses import dataclass, field -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest @@ -30,7 +30,6 @@ from tools.perf_tune_report.dcgm_correlate import ( DcgmCorrelateInputs, - PrometheusClient, TimeSeries, build_queries, correlate, @@ -41,7 +40,6 @@ write_correlation, ) - # --------------------------------------------------------------------------- # Fake Prometheus client # --------------------------------------------------------------------------- @@ -121,14 +119,14 @@ def query_range( }, }, "category_ceiling_map": { - "NCCL": {"metric": "nvlink5_tbps", "bound": "bandwidth"}, - "MoE": {"metric": "nvfp4_dense_pflops", "bound": "compute"}, - "FMHA": {"metric": "hbm3e_tbps", "bound": "bandwidth"}, - "BMM-NVFP4": {"metric": "nvfp4_dense_pflops", "bound": "compute"}, - "Triton-fused": {"metric": "bf16_dense_pflops", "bound": "compute"}, - "cuBLAS": {"metric": "bf16_dense_pflops", "bound": "compute"}, - "Elementwise": {"metric": "hbm3e_tbps", "bound": "bandwidth"}, - "Other": {"metric": "hbm3e_tbps", "bound": "bandwidth"}, + "NCCL": {"metric": "nvlink5_tbps", "bound": "bandwidth"}, + "MoE": {"metric": "nvfp4_dense_pflops", "bound": "compute"}, + "FMHA": {"metric": "hbm3e_tbps", "bound": "bandwidth"}, + "BMM-NVFP4": {"metric": "nvfp4_dense_pflops", "bound": "compute"}, + "Triton-fused": {"metric": "bf16_dense_pflops", "bound": "compute"}, + "cuBLAS": {"metric": "bf16_dense_pflops", "bound": "compute"}, + "Elementwise": {"metric": "hbm3e_tbps", "bound": "bandwidth"}, + "Other": {"metric": "hbm3e_tbps", "bound": "bandwidth"}, }, "dcgm_config": { "default_labels": ["gpu", "device"], @@ -162,7 +160,7 @@ def _series(metric: str, gpu: str, samples: list[tuple[float, float]]) -> TimeSe return TimeSeries(metric=metric, labels={"gpu": gpu}, samples=samples) -_START = datetime(2026, 5, 27, 13, 30, 0, tzinfo=timezone.utc) +_START = datetime(2026, 5, 27, 13, 30, 0, tzinfo=UTC) # --------------------------------------------------------------------------- @@ -248,11 +246,15 @@ def test_correlate_ratio_resource_happy_path(tmp_path): """ # DCGM_FI_PROF_DRAM_ACTIVE: 8 GPUs each at 0.5 across the sweep. dram_series = [ - _series("DCGM_FI_PROF_DRAM_ACTIVE", str(i), [ - (_START.timestamp(), 0.5), - (_START.timestamp() + 60, 0.5), - (_START.timestamp() + 120, 0.5), - ]) + _series( + "DCGM_FI_PROF_DRAM_ACTIVE", + str(i), + [ + (_START.timestamp(), 0.5), + (_START.timestamp() + 60, 0.5), + (_START.timestamp() + 120, 0.5), + ], + ) for i in range(8) ] client = _FakePromClient( @@ -279,9 +281,7 @@ def test_correlate_bytes_per_s_resource_happy_path(tmp_path): """ target_bps = 0.9e12 # 0.9 TB/s per GPU nvl_series = [ - _series("nvlink_combined", str(i), [ - (_START.timestamp() + t, target_bps) for t in range(0, 121, 15) - ]) + _series("nvlink_combined", str(i), [(_START.timestamp() + t, target_bps) for t in range(0, 121, 15)]) for i in range(8) ] client = _FakePromClient( @@ -322,10 +322,16 @@ def test_correlate_dry_run_emits_queries_without_executing(tmp_path): def test_correlate_short_sweep_warns(tmp_path): """30-second sweep is below the 60s min_sweep_seconds threshold.""" - dram = [_series("DCGM_FI_PROF_DRAM_ACTIVE", "0", [ - (_START.timestamp(), 0.3), - (_START.timestamp() + 30, 0.3), - ])] + dram = [ + _series( + "DCGM_FI_PROF_DRAM_ACTIVE", + "0", + [ + (_START.timestamp(), 0.3), + (_START.timestamp() + 30, 0.3), + ], + ) + ] client = _FakePromClient( instant_responses={"DCGM_FI_PROF_DRAM_ACTIVE": dram}, range_responses={"DCGM_FI_PROF_DRAM_ACTIVE": dram}, @@ -346,12 +352,16 @@ def test_correlate_short_sweep_warns(tmp_path): def test_read_sweep_window_happy_path(tmp_path): bundle = tmp_path / "bundle" bundle.mkdir() - (bundle / "inference_perfbench_v1.json").write_text(json.dumps({ - "bench": { - "captured_at": "2026-05-26T23:18:47Z", - "duration_effective_s": 400.5, - }, - })) + (bundle / "inference_perfbench_v1.json").write_text( + json.dumps( + { + "bench": { + "captured_at": "2026-05-26T23:18:47Z", + "duration_effective_s": 400.5, + }, + } + ) + ) start, end = read_sweep_window_from_bundle(bundle) assert start.tzinfo is not None assert (end - start).total_seconds() == pytest.approx(400.5) @@ -409,25 +419,27 @@ def test_write_correlation_writes_dcgm_correlation_json(tmp_path): } -def _make_workload_result(tmp_path: Path) -> "object": +def _make_workload_result(tmp_path: Path) -> object: """Helper: produce a populated DcgmCorrelationResult by running correlate against canned series for HBM (0.5 ratio) + NVLink (0.9 TB/s/GPU) + Tensor pipe (0.2 ratio = 1800 TFLOPS/GPU avg) across 120s on 8 GPUs.""" dram = [ - _series("DCGM_FI_PROF_DRAM_ACTIVE", str(i), - [(_START.timestamp(), 0.5), (_START.timestamp() + 60, 0.5), - (_START.timestamp() + 120, 0.5)]) + _series( + "DCGM_FI_PROF_DRAM_ACTIVE", + str(i), + [(_START.timestamp(), 0.5), (_START.timestamp() + 60, 0.5), (_START.timestamp() + 120, 0.5)], + ) for i in range(8) ] nvl = [ - _series("nvl_combined", str(i), - [(_START.timestamp() + t, 0.9e12) for t in range(0, 121, 15)]) - for i in range(8) + _series("nvl_combined", str(i), [(_START.timestamp() + t, 0.9e12) for t in range(0, 121, 15)]) for i in range(8) ] tensor = [ - _series("DCGM_FI_PROF_PIPE_TENSOR_ACTIVE", str(i), - [(_START.timestamp(), 0.2), (_START.timestamp() + 60, 0.2), - (_START.timestamp() + 120, 0.2)]) + _series( + "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE", + str(i), + [(_START.timestamp(), 0.2), (_START.timestamp() + 60, 0.2), (_START.timestamp() + 120, 0.2)], + ) for i in range(8) ] client = _FakePromClient( @@ -501,8 +513,7 @@ def test_correlate_with_kernels_json_path_populates_attribution(tmp_path): kj = tmp_path / "kernels.json" kj.write_text(json.dumps(_KERNELS_PAYLOAD)) - dram = [_series("DCGM_FI_PROF_DRAM_ACTIVE", "0", [(_START.timestamp(), 0.5), - (_START.timestamp() + 120, 0.5)])] + dram = [_series("DCGM_FI_PROF_DRAM_ACTIVE", "0", [(_START.timestamp(), 0.5), (_START.timestamp() + 120, 0.5)])] client = _FakePromClient( instant_responses={"DCGM_FI_PROF_DRAM_ACTIVE": dram}, range_responses={"DCGM_FI_PROF_DRAM_ACTIVE": dram}, @@ -535,9 +546,13 @@ def test_correlate_without_kernels_json_path_no_attribution(tmp_path): def test_correlate_captures_power_watts_per_gpu(tmp_path): """Live correlate() reads DCGM_FI_DEV_POWER_USAGE -> mean per-GPU watts.""" - power = [_series("DCGM_FI_DEV_POWER_USAGE", "0", - [(_START.timestamp(), 650.0), (_START.timestamp() + 15, 700.0), - (_START.timestamp() + 30, 750.0)])] + power = [ + _series( + "DCGM_FI_DEV_POWER_USAGE", + "0", + [(_START.timestamp(), 650.0), (_START.timestamp() + 15, 700.0), (_START.timestamp() + 30, 750.0)], + ) + ] client = _FakePromClient(range_responses={"DCGM_FI_DEV_POWER_USAGE": power}) inp = _inputs(tmp_path, start=_START, duration_s=120) @@ -601,26 +616,30 @@ def test_correlate_from_frozen_preserves_nodes(tmp_path): """A frozen YAML carrying nodes/namespace/pod_label_selector round-trips into the result so offline re-runs keep the re-query provenance.""" frozen = tmp_path / "frozen.yaml" - frozen.write_text(yaml.safe_dump({ - "schema_version": 1, - "hw_key": "b200_sm100", - "n_gpus": 8, - "sweep_start_utc": "2026-05-27T13:30:00Z", - "sweep_end_utc": "2026-05-27T13:32:00Z", - "data_tier": "prof", - "scrape_interval_s": 30.0, - "nodes": ["g55dc2e", "g04ade0"], - "namespace": "", - "pod_label_selector": "exported_pod=glm51-baseline-e2e-lat-x", - "resources": [ + frozen.write_text( + yaml.safe_dump( { - "peak_key": "hbm3e_tbps", - "metric": "DCGM_FI_PROF_DRAM_ACTIVE", - "unit": "ratio", - "measured_avg": 0.05, + "schema_version": 1, + "hw_key": "b200_sm100", + "n_gpus": 8, + "sweep_start_utc": "2026-05-27T13:30:00Z", + "sweep_end_utc": "2026-05-27T13:32:00Z", + "data_tier": "prof", + "scrape_interval_s": 30.0, + "nodes": ["g55dc2e", "g04ade0"], + "namespace": "", + "pod_label_selector": "exported_pod=glm51-baseline-e2e-lat-x", + "resources": [ + { + "peak_key": "hbm3e_tbps", + "metric": "DCGM_FI_PROF_DRAM_ACTIVE", + "unit": "ratio", + "measured_avg": 0.05, + } + ], } - ], - })) + ) + ) result = correlate_from_frozen(frozen, _CEILINGS) assert result.nodes == ["g55dc2e", "g04ade0"] assert result.namespace == "" @@ -658,13 +677,11 @@ def test_dcgm_correlation_json_includes_attribution_block(tmp_path): kj = tmp_path / "kernels.json" kj.write_text(json.dumps(_KERNELS_PAYLOAD)) - dram = [_series("DCGM_FI_PROF_DRAM_ACTIVE", "0", [(_START.timestamp(), 0.5), - (_START.timestamp() + 120, 0.5)])] + dram = [_series("DCGM_FI_PROF_DRAM_ACTIVE", "0", [(_START.timestamp(), 0.5), (_START.timestamp() + 120, 0.5)])] nvl = [_series("nvl", "0", [(_START.timestamp() + t, 0.9e12) for t in range(0, 121, 15)])] client = _FakePromClient( instant_responses={"DCGM_FI_PROF_DRAM_ACTIVE": dram}, - range_responses={"DCGM_FI_PROF_DRAM_ACTIVE": dram, - "DCGM_FI_PROF_NVLINK": nvl}, + range_responses={"DCGM_FI_PROF_DRAM_ACTIVE": dram, "DCGM_FI_PROF_NVLINK": nvl}, ) inp = _inputs(tmp_path, start=_START, duration_s=120) result = correlate(inp, _CEILINGS, client, kernels_json_path=kj) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_sol.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_sol.py index 1217667..e203927 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_sol.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_dcgm_sol.py @@ -32,7 +32,6 @@ ) from tools.perf_tune_report.schema import BACKEND_VLLM_SWEEP, STATUS_FULL, AtlasCell - # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -94,8 +93,13 @@ }, ], "queries": [ - {"peak_key": "hbm3e_tbps", "metric": "DCGM_FI_PROF_DRAM_ACTIVE", - "promql": "avg by (gpu) (DCGM_FI_PROF_DRAM_ACTIVE{...})", "unit": "ratio", "is_fallback": False}, + { + "peak_key": "hbm3e_tbps", + "metric": "DCGM_FI_PROF_DRAM_ACTIVE", + "promql": "avg by (gpu) (DCGM_FI_PROF_DRAM_ACTIVE{...})", + "unit": "ratio", + "is_fallback": False, + }, ], "dry_run": False, } @@ -148,6 +152,7 @@ def _count_pdf_pages(pdf_bytes: bytes) -> int: def test_render_page_raises_when_empty(): import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -162,6 +167,7 @@ def test_render_page_raises_when_empty(): def test_render_page_handles_no_resources(): """Payload exists but resources list is empty -> draws header + 'no resources' message.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -178,6 +184,7 @@ def test_render_page_handles_no_resources(): def test_render_page_handles_counter_fallback_caveat(): """group_level=counter triggers caveat text.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -192,6 +199,7 @@ def test_render_page_handles_counter_fallback_caveat(): def test_render_page_handles_short_sweep_caveat(): import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -338,6 +346,7 @@ def _attribution_payload() -> dict: def test_category_attribution_render_page_raises_when_empty_dict(): import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -352,6 +361,7 @@ def test_category_attribution_render_page_raises_when_empty_dict(): def test_category_attribution_render_page_raises_when_no_attribution(): """Payload exists but per_category_attribution is empty -> ValueError.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -367,14 +377,13 @@ def test_category_attribution_render_page_raises_when_no_attribution(): def test_category_attribution_render_page_happy_path(): import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt fig = plt.figure(figsize=(8, 11)) try: - dcgm_category_attribution.render_page( - fig, OrderedDict([("c1", _attribution_payload())]) - ) + dcgm_category_attribution.render_page(fig, OrderedDict([("c1", _attribution_payload())])) finally: plt.close(fig) @@ -382,22 +391,26 @@ def test_category_attribution_render_page_happy_path(): def test_category_attribution_render_handles_unmapped_category(): """Unknown category still renders (appended after canonical order).""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt payload = _attribution_payload() - payload["per_category_attribution"] = list(payload["per_category_attribution"]) + [{ - "category": "ExoticUnknownCategory", - "time_share_pct": 5.0, - "attributed_bytes_total": None, - "attributed_flops_total": None, - "effective_bw_during_category_window": None, - "effective_tflops_during_category_window": None, - "sol_pct_bw": None, - "sol_pct_compute": None, - "bound": None, - "ceiling_metric": None, - }] + payload["per_category_attribution"] = [ + *payload["per_category_attribution"], + { + "category": "ExoticUnknownCategory", + "time_share_pct": 5.0, + "attributed_bytes_total": None, + "attributed_flops_total": None, + "effective_bw_during_category_window": None, + "effective_tflops_during_category_window": None, + "sol_pct_bw": None, + "sol_pct_compute": None, + "bound": None, + "ceiling_metric": None, + }, + ] fig = plt.figure(figsize=(8, 11)) try: diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_experiments_index.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_experiments_index.py index 80d945b..f5b1e1e 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_experiments_index.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_experiments_index.py @@ -6,8 +6,6 @@ import sys from pathlib import Path -import pytest - REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) @@ -32,14 +30,13 @@ class _FakeS3List: def __init__(self, campaign_ids, page_size=2): self._keys = [ - f"perflake/perf-report/campaign_v1/dt=2026-05-31/campaign={c}/part-0.parquet" - for c in campaign_ids + f"perflake/perf-report/campaign_v1/dt=2026-05-31/campaign={c}/part-0.parquet" for c in campaign_ids ] self._page = page_size def list_objects_v2(self, **kw): token = int(kw.get("ContinuationToken", 0)) - chunk = self._keys[token:token + self._page] + chunk = self._keys[token : token + self._page] nxt = token + self._page more = nxt < len(self._keys) return { @@ -51,9 +48,9 @@ def list_objects_v2(self, **kw): def test_enumerate_published_campaign_ids_paginates(): ids = enumerate_published_campaign_ids( - cfg=None, bucket="perf-lake", - s3_client_factory=lambda _cfg: _FakeS3List(["a-20260531T010000Z", "b-20260531T020000Z", - "c-20260531T030000Z"]), + cfg=None, + bucket="perf-lake", + s3_client_factory=lambda _cfg: _FakeS3List(["a-20260531T010000Z", "b-20260531T020000Z", "c-20260531T030000Z"]), ) assert ids == {"a-20260531T010000Z", "b-20260531T020000Z", "c-20260531T030000Z"} @@ -73,8 +70,15 @@ def test_build_index_published_defaults_false_when_not_checked(tmp_path: Path): assert rows[0]["published_to_lake"] is False -def _stage(campaigns_root: Path, name: str, *, family: str = "", experiment_id: str | None = None, - rows: list[AtlasCell] | None = None, rendered: bool = True) -> Path: +def _stage( + campaigns_root: Path, + name: str, + *, + family: str = "", + experiment_id: str | None = None, + rows: list[AtlasCell] | None = None, + rendered: bool = True, +) -> Path: d = campaigns_root / name d.mkdir(parents=True) src = ["# Campaign", "", "- captured_at: x", "- config: /tmp/x.yaml"] @@ -86,27 +90,55 @@ def _stage(campaigns_root: Path, name: str, *, family: str = "", experiment_id: if rows is not None: write_jsonl(rows, d / "atlas.jsonl") if rendered: - (d / "report_status.json").write_text(json.dumps( - {"sol_complete": True, "plot_ready_points": 1, "omitted_pages": [], - "dcgm_grounded": True, "focus": "throughput", "sol_rigor": "L3"})) + (d / "report_status.json").write_text( + json.dumps( + { + "sol_complete": True, + "plot_ready_points": 1, + "omitted_pages": [], + "dcgm_grounded": True, + "focus": "throughput", + "sol_rigor": "L3", + } + ) + ) return d def _row(**kw) -> AtlasCell: - base = dict(cell_id="c1", model="GLM-5.1-NVFP4", hardware="B200", quant="NVFP4", - tensor_parallel=8, parallel_strategy="TP", mtp=False, - max_num_batched_tokens=8192, concurrency=1, status="full", - ttft_avg_ms=120.0, request_throughput_avg=0.2, output_tps_per_user=70.0, - output_tps_per_gpu=400.0, tpot_median_ms=13.0, backend="vllm-sweep") + base = dict( + cell_id="c1", + model="GLM-5.1-NVFP4", + hardware="B200", + quant="NVFP4", + tensor_parallel=8, + parallel_strategy="TP", + mtp=False, + max_num_batched_tokens=8192, + concurrency=1, + status="full", + ttft_avg_ms=120.0, + request_throughput_avg=0.2, + output_tps_per_user=70.0, + output_tps_per_gpu=400.0, + tpot_median_ms=13.0, + backend="vllm-sweep", + ) base.update(kw) return AtlasCell(**base) def test_build_index_row_reads_join_keys_and_headline(tmp_path: Path): - d = _stage(tmp_path, "glm51-nvfp4kv-ab-20260531T120000Z", family="nvfp4-kv", - experiment_id="glm51-nvfp4kv-ab-20260531T120000Z", - rows=[_row(output_tps_per_gpu=400.0, ttft_avg_ms=120.0), - _row(concurrency=8, output_tps_per_gpu=900.0, ttft_avg_ms=200.0)]) + d = _stage( + tmp_path, + "glm51-nvfp4kv-ab-20260531T120000Z", + family="nvfp4-kv", + experiment_id="glm51-nvfp4kv-ab-20260531T120000Z", + rows=[ + _row(output_tps_per_gpu=400.0, ttft_avg_ms=120.0), + _row(concurrency=8, output_tps_per_gpu=900.0, ttft_avg_ms=200.0), + ], + ) r = build_index_row(d) assert r["experiment_id"] == "glm51-nvfp4kv-ab-20260531T120000Z" assert r["family"] == "nvfp4-kv" @@ -129,8 +161,7 @@ def test_build_index_enumerates_and_sorts(tmp_path: Path): _stage(tmp_path, "a-20260531T010000Z", rows=[_row()]) (tmp_path / "not-a-campaign").mkdir() # ignored (no atlas/config/SOURCE) rows = build_index(tmp_path) - assert [r["campaign_id"] for r in rows] == [ - "a-20260531T010000Z", "b-20260531T020000Z"] + assert [r["campaign_id"] for r in rows] == ["a-20260531T010000Z", "b-20260531T020000Z"] def test_write_index_emits_jsonl_and_md(tmp_path: Path): @@ -154,9 +185,7 @@ def test_render_index_hides_absolute_bundle_prefix(tmp_path: Path): ) private_prefix = "/Users/operator/private/workspace" with (campaign / "SOURCE.md").open("a", encoding="utf-8") as source: - source.write( - f"- evidence_bundle_path: {private_prefix}/bundle-20260531T030000Z\n" - ) + source.write(f"- evidence_bundle_path: {private_prefix}/bundle-20260531T030000Z\n") markdown = render_index_md(build_index(tmp_path)) @@ -167,18 +196,28 @@ def test_render_index_hides_absolute_bundle_prefix(tmp_path: Path): def test_cli_experiments_index_family_filter(tmp_path: Path, capsys): _stage(tmp_path, "kv-20260531T040000Z", family="nvfp4-kv", rows=[_row()]) _stage(tmp_path, "ep-20260531T050000Z", family="deepep", rows=[_row()]) - rc = main(["experiments_index", "--campaigns-dir", str(tmp_path), - "--family", "nvfp4-kv", "--out", str(tmp_path / "o"), "--json"]) + rc = main( + [ + "experiments_index", + "--campaigns-dir", + str(tmp_path), + "--family", + "nvfp4-kv", + "--out", + str(tmp_path / "o"), + "--json", + ] + ) assert rc == 0 env = json.loads(capsys.readouterr().out) assert env["campaign_count"] == 1 - rows = [json.loads(l) for l in - (tmp_path / "o" / "experiments-index.jsonl").read_text().splitlines()] + rows = [json.loads(line) for line in (tmp_path / "o" / "experiments-index.jsonl").read_text().splitlines()] assert rows[0]["family"] == "nvfp4-kv" # --- experiment_inventory (canonical count: campaigns + bundles) ------------------------ + def _stage_bundle(root: Path, name: str, *, marker: str = "SOURCE.md") -> Path: """A run-id-stamped evidence bundle under a deploy-bundle-style tree.""" d = root / name @@ -250,8 +289,18 @@ def test_cli_experiment_inventory(tmp_path: Path, capsys): deploy = tmp_path / "x-deploy" _stage_bundle(deploy / "cluster-probes", "shared-20260531T010000Z") _stage_bundle(deploy / "experiments" / "artifacts", "extra-20260531T040000Z") - rc = main(["experiment_inventory", "--campaigns-dir", str(campaigns), - "--bundle-root", str(deploy), "--out", str(tmp_path / "o"), "--json"]) + rc = main( + [ + "experiment_inventory", + "--campaigns-dir", + str(campaigns), + "--bundle-root", + str(deploy), + "--out", + str(tmp_path / "o"), + "--json", + ] + ) assert rc == 0 env = json.loads(capsys.readouterr().out) assert env["total_experiments"] == 2 # shared (deduped) + extra-bundle diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_fleet_leaderboard.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_fleet_leaderboard.py index 9c159e1..0fbdd64 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_fleet_leaderboard.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_fleet_leaderboard.py @@ -23,11 +23,24 @@ def _row(**kw) -> AtlasCell: - base = dict(cell_id="c1", model="GLM-5.1-NVFP4", hardware="GB300", quant="NVFP4", - tensor_parallel=4, parallel_strategy="TP", mtp=False, - max_num_batched_tokens=8192, concurrency=1, status="full", - ttft_avg_ms=120.0, request_throughput_avg=0.2, output_tps_per_user=70.0, - output_tps_per_gpu=400.0, tpot_median_ms=13.0, backend="vllm-sweep") + base = dict( + cell_id="c1", + model="GLM-5.1-NVFP4", + hardware="GB300", + quant="NVFP4", + tensor_parallel=4, + parallel_strategy="TP", + mtp=False, + max_num_batched_tokens=8192, + concurrency=1, + status="full", + ttft_avg_ms=120.0, + request_throughput_avg=0.2, + output_tps_per_user=70.0, + output_tps_per_gpu=400.0, + tpot_median_ms=13.0, + backend="vllm-sweep", + ) base.update(kw) return AtlasCell(**base) @@ -68,25 +81,43 @@ def test_build_throughput_skips_failed_status(): def test_resolve_gpu_hr_override_and_config(tmp_path: Path): - from tools.perf_tune_report.fleet_leaderboard import resolve_gpu_hr, GPU_HR_DEFAULT + from tools.perf_tune_report.fleet_leaderboard import GPU_HR_DEFAULT, resolve_gpu_hr + cfg = tmp_path / "configs" cfg.mkdir() (cfg / "cost.yaml").write_text("usd_per_gpu_hour:\n GB300: 9.99\n default: 4.00\n") - assert resolve_gpu_hr("GB300", cfg, 12.0) == 12.0 # --gpu-hr override wins - assert resolve_gpu_hr("GB300", cfg) == 9.99 # per-hardware rate - assert resolve_gpu_hr("UNKNOWN-HW", cfg) == 4.00 # falls back to default + assert resolve_gpu_hr("GB300", cfg, 12.0) == 12.0 # --gpu-hr override wins + assert resolve_gpu_hr("GB300", cfg) == 9.99 # per-hardware rate + assert resolve_gpu_hr("UNKNOWN-HW", cfg) == 4.00 # falls back to default assert resolve_gpu_hr("GB300", tmp_path / "nope") == GPU_HR_DEFAULT # no config -> built-in default +def test_resolve_gpu_hr_ignores_non_mapping_yaml(tmp_path: Path): + from tools.perf_tune_report.fleet_leaderboard import GPU_HR_DEFAULT, resolve_gpu_hr + + cfg = tmp_path / "configs" + cfg.mkdir() + (cfg / "cost.yaml").write_text("- not\n- a mapping\n") + + assert resolve_gpu_hr("GB300", cfg) == GPU_HR_DEFAULT + + def test_build_quality_extracts_eval_acc_and_renders(): - from tools.perf_tune_report.fleet_leaderboard import build_quality, build_pareto, render_pareto_md + from tools.perf_tune_report.fleet_leaderboard import build_pareto, build_quality, render_pareto_md + eval_row = _row( - model="zai-org/GLM-5.1", cell_id="model-eval", concurrency=1, - ttft_avg_ms=None, request_throughput_avg=None, output_tps_per_user=None, - output_tps_per_gpu=None, tpot_median_ms=None, max_num_batched_tokens=0, - backend="aiperf", captured_at="2026-06-07T00:00:00Z", - extra={"metric_kind": "eval_acc", - "quality_metrics": {"gpqa_acc": 0.52, "mmlu_pro_acc": 0.903}}, + model="zai-org/GLM-5.1", + cell_id="model-eval", + concurrency=1, + ttft_avg_ms=None, + request_throughput_avg=None, + output_tps_per_user=None, + output_tps_per_gpu=None, + tpot_median_ms=None, + max_num_batched_tokens=0, + backend="aiperf", + captured_at="2026-06-07T00:00:00Z", + extra={"metric_kind": "eval_acc", "quality_metrics": {"gpqa_acc": 0.52, "mmlu_pro_acc": 0.903}}, ) q = build_quality([eval_row]) assert q["GLM-5.1"]["gpqa_acc"] == 0.52 @@ -101,10 +132,10 @@ def test_build_quality_extracts_eval_acc_and_renders(): def test_build_aa_latest_captured_wins(): rows = [ - _row(model="A", cell_id="aa-1k", concurrency=1, output_tps_per_user=100.0, - captured_at="2026-06-01T00:00:00Z"), - _row(model="A", cell_id="aa-1k", concurrency=1, output_tps_per_user=222.0, - captured_at="2026-06-05T00:00:00Z"), # newer wins + _row(model="A", cell_id="aa-1k", concurrency=1, output_tps_per_user=100.0, captured_at="2026-06-01T00:00:00Z"), + _row( + model="A", cell_id="aa-1k", concurrency=1, output_tps_per_user=222.0, captured_at="2026-06-05T00:00:00Z" + ), # newer wins ] aa = build_aa(rows) assert aa["A"][("aa-1k", 1)]["opu"] == 222.0 @@ -117,6 +148,7 @@ def model(name, opu, ttft, tps): _row(model=name, cell_id="aa-1k", concurrency=1, output_tps_per_user=opu, ttft_avg_ms=ttft), _row(model=name, cell_id="sweep", concurrency=256, tensor_parallel=4, output_tps_per_gpu=tps), ] + rows = model("AAA", 200, 100, 2000) + model("BBB", 100, 300, 1000) + model("CCC", 250, 50, 500) M = build_pareto(rows) assert set(M) == {"AAA", "BBB", "CCC"} @@ -142,10 +174,10 @@ def test_read_all_rows_filters_hardware(tmp_path: Path): def test_write_leaderboards_emits_three_files(tmp_path: Path): - rows = ( - [_row(model="Fast", cell_id="aa-1k", concurrency=1, output_tps_per_user=220.0, ttft_avg_ms=100.0), - _row(model="Fast", cell_id="sweep", concurrency=256, output_tps_per_gpu=2900.0)] - ) + rows = [ + _row(model="Fast", cell_id="aa-1k", concurrency=1, output_tps_per_user=220.0, ttft_avg_ms=100.0), + _row(model="Fast", cell_id="sweep", concurrency=256, output_tps_per_gpu=2900.0), + ] out = tmp_path / "out" res = write_leaderboards(rows, out, hw="GB300", gpu_hr=8.60) for key in ("aa", "throughput", "pareto"): @@ -156,12 +188,15 @@ def test_write_leaderboards_emits_three_files(tmp_path: Path): def test_cli_fleet_leaderboard_smoke(tmp_path: Path, capsys): campaigns = tmp_path / "campaigns" - _stage(campaigns, "camp-20260607T000000Z", [ - _row(model="Champ", cell_id="aa-1k", concurrency=1, output_tps_per_user=221.0, ttft_avg_ms=170.0), - _row(model="Champ", cell_id="sweep", concurrency=160, output_tps_per_gpu=2916.0), - ]) - rc = main(["fleet_leaderboard", "--campaigns-dir", str(campaigns), - "--out", str(tmp_path / "out"), "--json"]) + _stage( + campaigns, + "camp-20260607T000000Z", + [ + _row(model="Champ", cell_id="aa-1k", concurrency=1, output_tps_per_user=221.0, ttft_avg_ms=170.0), + _row(model="Champ", cell_id="sweep", concurrency=160, output_tps_per_gpu=2916.0), + ], + ) + rc = main(["fleet_leaderboard", "--campaigns-dir", str(campaigns), "--out", str(tmp_path / "out"), "--json"]) assert rc == 0 assert (tmp_path / "out" / "FLEET-MODEL-SELECTION-GB300.md").is_file() assert "Champ" in (tmp_path / "out" / "THROUGHPUT-FLEET-LEADERBOARD-GB300.md").read_text() diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_graph_diff.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_graph_diff.py index 54554db..a24a16d 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_graph_diff.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_graph_diff.py @@ -14,7 +14,6 @@ extract_inductor_passes, ) - # --------------------------------------------------------------------------- # Synthetic log fixtures # --------------------------------------------------------------------------- @@ -225,6 +224,7 @@ def forward(x): return x def test_cli_graph_diff_smoke(tmp_path: Path) -> None: """End-to-end via the CLI command handler.""" import argparse + from tools.perf_tune_report.perf_tune_report_cli import cmd_graph_diff a, b = _write_logs(tmp_path) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_aiperf.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_aiperf.py index a6d552a..c17570e 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_aiperf.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_aiperf.py @@ -7,13 +7,13 @@ import pytest +from tools.perf_tune_report.importers import import_bundle_auto from tools.perf_tune_report.importers.aiperf_export import ( AiperfImportResult, + _parse_aiperf_csv, detect_aiperf_bundle, import_aiperf_bundle, - _parse_aiperf_csv, ) -from tools.perf_tune_report.importers import import_bundle_auto # Minimal AIPerf profile_export_aiperf.csv (two sections), matching the real # 2026-05-31 campaign export shape. @@ -65,9 +65,7 @@ def test_import_and_status(tmp_path: Path) -> None: v = _make_variant(tmp_path) campaign = tmp_path / "campaign" campaign.mkdir() - res = import_aiperf_bundle( - v, campaign, overrides={"model": "zai-org/GLM-5.1", "mtp": True} - ) + res = import_aiperf_bundle(v, campaign, overrides={"model": "zai-org/GLM-5.1", "mtp": True}) assert isinstance(res, AiperfImportResult) assert res.cell_id == "mtpk3" assert res.concurrencies == [1, 8] @@ -125,7 +123,8 @@ def test_aiperf_isl_osl_total_prefix_cache_and_cache_mode(tmp_path: Path) -> Non campaign = tmp_path / "campaign" campaign.mkdir() res = import_aiperf_bundle( - v, campaign, + v, + campaign, overrides={"model": "zai-org/GLM-5.1", "tensor_parallel": 8, "cache_mode": "warm"}, ) row = json.loads(res.normalized_path.read_text())[0] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_drive_load.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_drive_load.py index 7138f27..78232e7 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_drive_load.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_drive_load.py @@ -18,7 +18,6 @@ import_drive_load_bundle, ) - # --------------------------------------------------------------------------- # Fixture helpers # --------------------------------------------------------------------------- @@ -94,9 +93,7 @@ def _make_multi_c_bundle(tmp_path: Path, *, with_meta: bool = True) -> Path: # ok-rate scales inversely with concurrency to mimic a stress sweep n_ok = {15: 60, 30: 40, 60: 30}[c] n_fail = {15: 0, 30: 4, 60: 6}[c] - (sub / "load.jsonl").write_text( - _make_jsonl_lines(n_ok, n_fail, sweep_offset=c * 100.0) - ) + (sub / "load.jsonl").write_text(_make_jsonl_lines(n_ok, n_fail, sweep_offset=c * 100.0)) if with_meta: (bundle / "inference_perfbench_v1.json").write_text(json.dumps(_BUNDLE_META)) return bundle @@ -340,9 +337,7 @@ def test_import_dry_run_writes_nothing(tmp_path: Path) -> None: def test_import_single_c_with_override(tmp_path: Path) -> None: bundle = _make_single_c_bundle(tmp_path) campaign = _make_campaign(tmp_path) - result = import_drive_load_bundle( - bundle, campaign, concurrency_override=15 - ) + result = import_drive_load_bundle(bundle, campaign, concurrency_override=15) assert result.concurrencies == [15] rows = json.loads(result.normalized_path.read_text()) assert len(rows) == 1 @@ -354,7 +349,9 @@ def test_import_drive_load_isl_osl_total_and_cache_mode(tmp_path: Path) -> None: bundle = _make_single_c_bundle(tmp_path) campaign = _make_campaign(tmp_path) result = import_drive_load_bundle( - bundle, campaign, concurrency_override=15, + bundle, + campaign, + concurrency_override=15, overrides={"cache_mode": "cold", "tensor_parallel": 8}, ) row = json.loads(result.normalized_path.read_text())[0] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_perf_bench.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_perf_bench.py index 6274a05..28a6688 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_perf_bench.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_perf_bench.py @@ -9,16 +9,17 @@ from tools.perf_tune_report.capture_signature import variant_key_for from tools.perf_tune_report.importers.inference_perf_bench import ( + _SWEEP_K, + _SWEEP_SIMPLE, ImportResult, + _CellIdentity, _enumerate_sweep_files, _parse_metrics, - _SWEEP_K, - _SWEEP_SIMPLE, + _row_from_metrics, import_perf_bench_bundle, ) from tools.perf_tune_report.schema import AtlasCell - # --- sample bench-serve output blocks (anonymized; matches V1 sweep format) - _SWEEP_C8 = """============ Serving Benchmark Result ============ @@ -123,9 +124,15 @@ def test_enumerate_sweep_files_sorted_by_k_then_c(tmp_path): (raw / f"sweep-K{k}-c{c}.txt").write_text(_SWEEP_C8) files = _enumerate_sweep_files(bundle) assert [(f.k, f.concurrency) for f in files] == [ - (1, 1), (1, 4), (1, 16), - (2, 1), (2, 4), (2, 16), - (3, 1), (3, 4), (3, 16), + (1, 1), + (1, 4), + (1, 16), + (2, 1), + (2, 4), + (2, 16), + (3, 1), + (3, 4), + (3, 16), ] @@ -195,9 +202,7 @@ def test_import_derives_mean_isl_osl(tmp_path): def test_import_cache_mode_override(tmp_path): bundle = _make_bundle(tmp_path) campaign = _make_campaign(tmp_path) - import_perf_bench_bundle( - bundle, campaign, overrides={"cell_id": "c", "cache_mode": "warm"} - ) + import_perf_bench_bundle(bundle, campaign, overrides={"cell_id": "c", "cache_mode": "warm"}) rows = json.loads((campaign / "cells" / "c" / "normalized.json").read_text()) assert all(r["cache_mode"] == "warm" for r in rows) @@ -215,9 +220,7 @@ def test_import_prefix_cache_hit_rate_from_bundle_meta(tmp_path): raw = bundle / "raw" raw.mkdir(parents=True) (raw / "sweep-c8.txt").write_text(_SWEEP_C8) - (bundle / "inference_perfbench_v1.json").write_text( - json.dumps({"model": "m", "prefix_cache_hit_rate": 0.83}) - ) + (bundle / "inference_perfbench_v1.json").write_text(json.dumps({"model": "m", "prefix_cache_hit_rate": 0.83})) campaign = _make_campaign(tmp_path) import_perf_bench_bundle(bundle, campaign, overrides={"cell_id": "c"}) rows = json.loads((campaign / "cells" / "c" / "normalized.json").read_text()) @@ -350,17 +353,24 @@ def test_import_k_sweep_emits_k_suffixed_cell_ids(tmp_path): for c in (1, 4): (raw / f"sweep-K{k}-c{c}.txt").write_text(_SWEEP_C8) (bundle / "inference_perfbench_v1.json").write_text( - json.dumps({"model": "deepseek-ai/DeepSeek-V4-Flash", "hardware": "B200", - "quant": "NVFP4", "tensor_parallel_size": 8, - "parallel_strategy": "TP", "mtp": True, - "max_num_batched_tokens": 12288}) + json.dumps( + { + "model": "deepseek-ai/DeepSeek-V4-Flash", + "hardware": "B200", + "quant": "NVFP4", + "tensor_parallel_size": 8, + "parallel_strategy": "TP", + "mtp": True, + "max_num_batched_tokens": 12288, + } + ) ) campaign = _make_campaign(tmp_path) result = import_perf_bench_bundle(bundle, campaign) data = json.loads(result.normalized_path.read_text()) - # 3 K values × 2 concurrencies = 6 rows + # 3 K values x 2 concurrencies = 6 rows assert len(data) == 6 assert result.k_values == [1, 2, 3] # K=1 keeps the base cell_id; K>1 gets a -K suffix @@ -374,12 +384,46 @@ def test_import_k_sweep_emits_k_suffixed_cell_ids(tmp_path): assert k3_row["extra"]["spec_decode_k"] == 3 +def test_variant_extra_does_not_overwrite_k_suffix(tmp_path): + identity = _CellIdentity( + cell_id="cell", + model="model", + hardware="B200", + quant="FP8", + tensor_parallel=8, + parallel_strategy="TP", + mtp=True, + max_num_batched_tokens=4096, + variant_extra={"custom_knob": "enabled"}, + ) + + row = _row_from_metrics( + identity, + concurrency=4, + metrics={ + "req_per_s": 1.0, + "output_tps": 100.0, + "total_tps": 200.0, + "ttft_med_ms": 50.0, + "tpot_med_ms": 10.0, + }, + sweep_path=tmp_path / "raw" / "sweep-K3-c4.txt", + k=3, + captured_at="2026-08-16T00:00:00Z", + ) + + assert row.cell_id == "cell-K3" + assert row.extra["spec_decode_k"] == 3 + assert row.extra["custom_knob"] == "enabled" + + def test_import_overrides_take_precedence_over_bundle_meta(tmp_path): bundle = _make_bundle(tmp_path) campaign = _make_campaign(tmp_path) result = import_perf_bench_bundle( - bundle, campaign, + bundle, + campaign, overrides={ "cell_id": "operator-named-cell", "model": "operator/override-model", @@ -410,7 +454,8 @@ def test_import_no_metadata_with_overrides_works(tmp_path): bundle = _make_bundle(tmp_path, with_meta=False) campaign = _make_campaign(tmp_path) result = import_perf_bench_bundle( - bundle, campaign, + bundle, + campaign, overrides={ "model": "moonshotai/Kimi-K2.6", "quant": "NVFP4", diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_variant_ab.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_variant_ab.py index 500ab36..4417b8a 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_variant_ab.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_variant_ab.py @@ -12,6 +12,7 @@ detect_variant_ab, import_variant_ab_bundle, ) +from tools.perf_tune_report.importers.zymtrace_kernels import ZymtraceTSVMissing def _bench_text(*, n, reqps, out_tps, total_tps, ttft, tpot) -> str: @@ -48,7 +49,9 @@ def ab_bundle(tmp_path: Path) -> Path: bundle = tmp_path / "ab-c32" # arm 1: base -- 2 concurrencies x 2 trials (averaged) _make_arm( - bundle / "minimax-m27-v-base", tp=4, warm=True, + bundle / "minimax-m27-v-base", + tp=4, + warm=True, by_c_trials={ 1: [(8, 0.21, 220.0, 1100.0, 130.0, 4.40), (8, 0.21, 226.0, 1120.0, 142.0, 4.42)], 32: [(96, 2.80, 2890.0, 14400.0, 640.0, 10.20), (96, 2.86, 3100.0, 15600.0, 660.0, 9.40)], @@ -56,7 +59,9 @@ def ab_bundle(tmp_path: Path) -> Path: ) # arm 2: ep _make_arm( - bundle / "minimax-m27-v-ep", tp=4, warm=True, + bundle / "minimax-m27-v-ep", + tp=4, + warm=True, by_c_trials={ 32: [(96, 2.79, 2880.0, 14300.0, 650.0, 10.30), (96, 2.85, 3090.0, 15500.0, 655.0, 9.50)], }, @@ -73,9 +78,15 @@ def test_import_emits_plot_ready_cells(ab_bundle, tmp_path): campaign = tmp_path / "campaign" (campaign / "cells").mkdir(parents=True) res = import_variant_ab_bundle( - ab_bundle, campaign, - overrides={"model": "nvidia/MiniMax-M2.7-NVFP4", "hardware": "GB300", - "quant": "NVFP4", "tensor_parallel": 4, "max_num_batched_tokens": 8192}, + ab_bundle, + campaign, + overrides={ + "model": "nvidia/MiniMax-M2.7-NVFP4", + "hardware": "GB300", + "quant": "NVFP4", + "tensor_parallel": 4, + "max_num_batched_tokens": 8192, + }, captured_at="2026-06-05T00:00:00Z", ) assert sorted(res.cells) == ["minimax-m27-v-base", "minimax-m27-v-ep"] @@ -101,7 +112,8 @@ def test_auto_dispatch_routes_to_variant_ab(ab_bundle, tmp_path): campaign = tmp_path / "campaign2" (campaign / "cells").mkdir(parents=True) res = import_bundle_auto( - ab_bundle, campaign, + ab_bundle, + campaign, overrides={"model": "nvidia/MiniMax-M2.7-NVFP4", "hardware": "GB300"}, ) assert getattr(res, "importer", None) == "variant_ab" @@ -112,9 +124,9 @@ def test_auto_dispatch_routes_to_variant_ab(ab_bundle, tmp_path): def test_mtp_inferred_from_arm_name(tmp_path): bundle = tmp_path / "ab" - _make_arm(bundle / "m-v-mtpk3", tp=4, warm=True, - by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) - campaign = tmp_path / "c"; (campaign / "cells").mkdir(parents=True) + _make_arm(bundle / "m-v-mtpk3", tp=4, warm=True, by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) + campaign = tmp_path / "c" + (campaign / "cells").mkdir(parents=True) import_variant_ab_bundle(bundle, campaign, overrides={"model": "x", "hardware": "GB300"}) row = json.loads((campaign / "cells" / "m-v-mtpk3" / "normalized.json").read_text())[0] assert row["mtp"] is True @@ -140,7 +152,8 @@ def test_arm_result_json_flows_variant_descriptors(tmp_path): "flags": ["--enable-expert-parallel"], }, ) - campaign = tmp_path / "c"; (campaign / "cells").mkdir(parents=True) + campaign = tmp_path / "c" + (campaign / "cells").mkdir(parents=True) import_variant_ab_bundle(bundle, campaign, overrides={"model": "x", "hardware": "GB300"}) row = json.loads((campaign / "cells" / "qnext-v-fimoe" / "normalized.json").read_text())[0] assert row["parallel_strategy"] == "EP" @@ -163,14 +176,9 @@ def test_arm_result_json_flows_variant_descriptors(tmp_path): _ZYM_TSVS = { "kernel-class.tsv": "event_kind\tkind\tsamples\ncuda\tnative\t1000\ncuda\tcuda\t500\n", "top-gpu-frames.tsv": ( - "kernel\tsamples\n" - "multimem_all_reduce_kernel\t558\n" - "bmm_E2m1E2m1_Fp32_sm100f\t328\n" - ), - "per-gpu.tsv": ( - "gpu_name\tgpu_uuid\tsamples\n" - "NVIDIA GB300\taaaa\t35961\n" + "kernel\tsamples\nmultimem_all_reduce_kernel\t558\nbmm_E2m1E2m1_Fp32_sm100f\t328\n" ), + "per-gpu.tsv": ("gpu_name\tgpu_uuid\tsamples\nNVIDIA GB300\taaaa\t35961\n"), "per-category.tsv": "category\tsamples\nBMM-NVFP4\t19212\nFMHA\t9226\n", "top-python-during-cuda.tsv": "python_frame\tsamples\nvllm.x\t12345\n", } @@ -180,8 +188,7 @@ def _add_zymtrace(arm_dir: Path) -> None: """Drop a valid capture_sources.json + zymtrace/*.tsv into an arm dir, as the hardened run-variant-ab.sh inline SoL capture does before teardown.""" (arm_dir / "capture_sources.json").write_text( - json.dumps({"schema_version": 1, "captured_sources": ["zymtrace"], - "pod_name": arm_dir.name}) + json.dumps({"schema_version": 1, "captured_sources": ["zymtrace"], "pod_name": arm_dir.name}) ) zd = arm_dir / "zymtrace" zd.mkdir() @@ -192,10 +199,10 @@ def _add_zymtrace(arm_dir: Path) -> None: def test_zymtrace_kernels_emitted_when_arm_declares_it(tmp_path): bundle = tmp_path / "ab" arm = bundle / "minimax-m27-v-ep" - _make_arm(arm, tp=4, warm=True, - by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) + _make_arm(arm, tp=4, warm=True, by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) _add_zymtrace(arm) - campaign = tmp_path / "c"; (campaign / "cells").mkdir(parents=True) + campaign = tmp_path / "c" + (campaign / "cells").mkdir(parents=True) import_variant_ab_bundle(bundle, campaign, overrides={"model": "x", "hardware": "GB300"}) kj = campaign / "cells" / "minimax-m27-v-ep" / "kernels.json" assert kj.is_file(), "kernels.json must be emitted for an arm declaring zymtrace" @@ -208,9 +215,9 @@ def test_no_kernels_json_when_arm_has_no_manifest(tmp_path): # An arm WITHOUT capture_sources.json (single-engine / pre-SoL bundle) must still # import cleanly and simply not emit kernels.json (correct no-op). bundle = tmp_path / "ab" - _make_arm(bundle / "m-v-base", tp=4, warm=True, - by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) - campaign = tmp_path / "c"; (campaign / "cells").mkdir(parents=True) + _make_arm(bundle / "m-v-base", tp=4, warm=True, by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) + campaign = tmp_path / "c" + (campaign / "cells").mkdir(parents=True) res = import_variant_ab_bundle(bundle, campaign, overrides={"model": "x", "hardware": "GB300"}) assert res.status == "full" assert not (campaign / "cells" / "m-v-base" / "kernels.json").exists() @@ -220,10 +227,10 @@ def test_declared_but_broken_zymtrace_aborts_import(tmp_path): # Manifest declares zymtrace but a TSV is 0-byte -> loud failure (no silent degrade). bundle = tmp_path / "ab" arm = bundle / "m-v-ep" - _make_arm(arm, tp=4, warm=True, - by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) + _make_arm(arm, tp=4, warm=True, by_c_trials={32: [(96, 2.8, 2900.0, 14000.0, 640.0, 10.0)]}) _add_zymtrace(arm) (arm / "zymtrace" / "per-category.tsv").write_text("") # 0-byte -> ZymtraceTSVMissing - campaign = tmp_path / "c"; (campaign / "cells").mkdir(parents=True) - with pytest.raises(Exception): + campaign = tmp_path / "c" + (campaign / "cells").mkdir(parents=True) + with pytest.raises(ZymtraceTSVMissing): import_variant_ab_bundle(bundle, campaign, overrides={"model": "x", "hardware": "GB300"}) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_workloads.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_workloads.py index 16e249d..25e58ce 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_workloads.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_import_workloads.py @@ -33,9 +33,7 @@ def _bench_dir(tmp_path: Path, files: dict[str, str], workloads: list[dict]) -> bench.mkdir() for name, text in files.items(): (bench / name).write_text(text) - (bench / "bench-workloads.json").write_text( - json.dumps({"schema": "bench_workloads_v1", "workloads": workloads}) - ) + (bench / "bench-workloads.json").write_text(json.dumps({"schema": "bench_workloads_v1", "workloads": workloads})) return bench @@ -56,8 +54,14 @@ def test_import_workloads_tags_dataset_and_isl_osl(tmp_path): ) camp = _campaign(tmp_path) res = import_workloads( - bench, camp, model="GLM-5.1", hardware="GB300", tensor_parallel=4, - kv_cache_dtype="fp8_e4m3", image="infr/vllm:v2.12.3", max_num_batched_tokens=2048, + bench, + camp, + model="GLM-5.1", + hardware="GB300", + tensor_parallel=4, + kv_cache_dtype="fp8_e4m3", + image="infr/vllm:v2.12.3", + max_num_batched_tokens=2048, ) assert res.n_cells == 2 assert set(res.tags) == {"sonnet", "random"} @@ -80,23 +84,26 @@ def test_import_workloads_tags_dataset_and_isl_osl(tmp_path): def test_import_workloads_measured_mean_overrides_nominal(tmp_path): # When the bench output carries token totals, the MEASURED mean wins over the nominal. - bench_text = _BENCH + "Total input tokens: 6400\n" \ - "Total generated tokens: 3200\n" + bench_text = ( + _BENCH + "Total input tokens: 6400\nTotal generated tokens: 3200\n" + ) bench = _bench_dir( - tmp_path, {"random-c1.txt": bench_text}, + tmp_path, + {"random-c1.txt": bench_text}, [{"tag": "random", "dataset": "random", "isl": 1024, "osl": 512}], ) camp = _campaign(tmp_path) import_workloads(bench, camp, model="M", hardware="GB300", tensor_parallel=4) r = json.loads((camp / "cells" / "random" / "normalized.json").read_text())[0] - assert r["mean_input_tokens"] == pytest.approx(6400 / 64) # measured, not 1024 nominal + assert r["mean_input_tokens"] == pytest.approx(6400 / 64) # measured, not 1024 nominal assert r["mean_output_tokens"] == pytest.approx(3200 / 64) def test_import_workloads_multihyphen_tag(tmp_path): # aa-1k tag (multi-hyphen) must resolve to tag=aa-1k, c=1 (greedy anchor on last -c). bench = _bench_dir( - tmp_path, {"aa-1k-c1.txt": _BENCH}, + tmp_path, + {"aa-1k-c1.txt": _BENCH}, [{"tag": "aa-1k", "dataset": "aa", "isl": 1000, "osl": 1000}], ) camp = _campaign(tmp_path) @@ -108,12 +115,12 @@ def test_import_workloads_multihyphen_tag(tmp_path): def test_import_workloads_dry_run_writes_nothing(tmp_path): bench = _bench_dir( - tmp_path, {"sonnet-c1.txt": _BENCH}, + tmp_path, + {"sonnet-c1.txt": _BENCH}, [{"tag": "sonnet", "dataset": "sonnet", "isl": 512, "osl": 256}], ) camp = _campaign(tmp_path) - res = import_workloads(bench, camp, model="M", hardware="GB300", tensor_parallel=4, - dry_run=True) + res = import_workloads(bench, camp, model="M", hardware="GB300", tensor_parallel=4, dry_run=True) assert res.n_cells == 1 and res.n_rows == 1 assert not (camp / "cells" / "sonnet").exists() @@ -126,21 +133,37 @@ def test_import_workloads_missing_workloads_json_defaults_dataset_to_tag(tmp_pat camp = _campaign(tmp_path) import_workloads(bench, camp, model="M", hardware="GB300", tensor_parallel=4) r = json.loads((camp / "cells" / "sonnet" / "normalized.json").read_text())[0] - assert r["dataset"] == "sonnet" # tag is the fallback dataset - assert r["mean_input_tokens"] is None # no nominal, no totals + assert r["dataset"] == "sonnet" # tag is the fallback dataset + assert r["mean_input_tokens"] is None # no nominal, no totals def test_cli_import_workloads(tmp_path, capsys): bench = _bench_dir( - tmp_path, {"sonnet-c1.txt": _BENCH}, + tmp_path, + {"sonnet-c1.txt": _BENCH}, [{"tag": "sonnet", "dataset": "sonnet", "isl": 512, "osl": 256}], ) camp = _campaign(tmp_path, "campaigns/mycamp") - rc = main([ - "import_workloads", "--bench-dir", str(bench), "--campaign", str(camp), - "--model", "GLM-5.1", "--hardware", "GB300", "--tensor-parallel", "4", - "--kv-cache-dtype", "fp8_e4m3", "--image", "infr/vllm:v2.12.3", "--json", - ]) + rc = main( + [ + "import_workloads", + "--bench-dir", + str(bench), + "--campaign", + str(camp), + "--model", + "GLM-5.1", + "--hardware", + "GB300", + "--tensor-parallel", + "4", + "--kv-cache-dtype", + "fp8_e4m3", + "--image", + "infr/vllm:v2.12.3", + "--json", + ] + ) assert rc == 0 env = json.loads(capsys.readouterr().out) assert env["verb"] == "import_workloads" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_profile.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_profile.py index d150b26..b6d1004 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_profile.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_profile.py @@ -15,7 +15,6 @@ capture_kernel_profile, ) - # --------------------------------------------------------------------------- # Fake step-fn implementation for testing — records calls + returns canned data # --------------------------------------------------------------------------- @@ -55,9 +54,7 @@ def attach_sidecar( self._maybe_fail("attach_sidecar") return {"cmd": f"FAKE kubectl debug {pod}", "sidecar_container": sidecar_container} - def wait_for_sidecar( - self, *, namespace: str, pod: str, sidecar_container: str - ) -> dict[str, Any]: + def wait_for_sidecar(self, *, namespace: str, pod: str, sidecar_container: str) -> dict[str, Any]: self.calls.append(("wait_for_sidecar", locals().copy())) self._maybe_fail("wait_for_sidecar") return {"cmd": "FAKE poll", "state": "running", "ready_after_s": 2} @@ -341,6 +338,7 @@ def test_custom_nsys_args_passed_to_step(tmp_path: Path) -> None: def test_cli_kernel_profile_refuses_without_ack(tmp_path: Path, capsys) -> None: """Live invocation without ack flag returns FATAL (does not run kubectl).""" import argparse + from tools.perf_tune_report.perf_tune_report_cli import cmd_kernel_profile ns = argparse.Namespace( diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_reproducer.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_reproducer.py index a2bc740..408428f 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_reproducer.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_kernel_reproducer.py @@ -1,4 +1,5 @@ """Tests for the perf_tune_report kernel_reproducer_scaffold verb.""" + from __future__ import annotations from pathlib import Path @@ -70,9 +71,12 @@ def test_cli_verb_registered() -> None: ns = parser.parse_args( [ "kernel_reproducer_scaffold", - "--kernel-name", "linear_sm100_mpk_task_impl", - "--header", "tasks/blackwell/linear_sm100_mpk.cuh", - "--output-dir", "/tmp/x", + "--kernel-name", + "linear_sm100_mpk_task_impl", + "--header", + "tasks/blackwell/linear_sm100_mpk.cuh", + "--output-dir", + "/tmp/x", "--dry-run", "--json", ] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_lws_summary.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_lws_summary.py index ca1ac07..f3ffca5 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_lws_summary.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_lws_summary.py @@ -101,9 +101,7 @@ def test_detect_lws_summary_empty_variants(tmp_path: Path): def test_detect_lws_summary_variant_missing_metrics_per_concurrency(tmp_path: Path): - (tmp_path / "summary.json").write_text( - json.dumps({"variants": [{"short": "x", "label": "x"}]}) - ) + (tmp_path / "summary.json").write_text(json.dumps({"variants": [{"short": "x", "label": "x"}]})) assert detect_lws_summary(tmp_path) is False @@ -169,14 +167,12 @@ def test_import_lws_summary_max_num_batched_tokens_parsed_from_knob(tmp_path: Pa def test_import_lws_summary_raises_when_summary_missing(tmp_path: Path): - with pytest.raises(ValueError, match="summary.json not found"): + with pytest.raises(ValueError, match=r"summary\.json not found"): import_lws_summary_bundle(bundle=tmp_path, campaign_dir=tmp_path) def test_import_lws_summary_raises_when_no_rows_emitted(tmp_path: Path): - (tmp_path / "summary.json").write_text( - json.dumps({"variants": [{"short": "x", "metrics_per_concurrency": []}]}) - ) + (tmp_path / "summary.json").write_text(json.dumps({"variants": [{"short": "x", "metrics_per_concurrency": []}]})) with pytest.raises(ValueError, match="no rows emitted"): import_lws_summary_bundle(bundle=tmp_path, campaign_dir=tmp_path) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_methodology_gate.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_methodology_gate.py index f77dd2e..2cc1abc 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_methodology_gate.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_methodology_gate.py @@ -136,23 +136,32 @@ def test_methodology_missing_gmu_flagged(): def test_methodology_missing_isl_osl_flagged(): """A vllm-bench (non-aa) measured row must record per-request ISL/OSL -- the workload shape, not just max_num_batched_tokens (docs/METHODOLOGY.md).""" - rows = [_make_atlas_row(cell_id="cellA", cache_mode="cold", - mean_input_tokens=None, mean_output_tokens=None)] + rows = [_make_atlas_row(cell_id="cellA", cache_mode="cold", mean_input_tokens=None, mean_output_tokens=None)] p = methodology_problems(rows) assert any("mean_input_tokens/mean_output_tokens" in x and "cellA" in x for x in p) def test_methodology_isl_osl_exempt_aa_dataset(): """aa-* workloads define their shape by the dataset name -> ISL/OSL exempt.""" - rows = [_make_atlas_row(cell_id="cellA", cache_mode="cold", dataset="aa-10k", - mean_input_tokens=None, mean_output_tokens=None)] + rows = [ + _make_atlas_row( + cell_id="cellA", cache_mode="cold", dataset="aa-10k", mean_input_tokens=None, mean_output_tokens=None + ) + ] assert methodology_problems(rows) == [] def test_methodology_isl_osl_exempt_aiperf_backend(): """aiperf / drive_load backends legitimately leave ISL/OSL None -> exempt.""" - rows = [_make_atlas_row(cell_id="cellA", cache_mode="cold", bench_backend="drive_load", - mean_input_tokens=None, mean_output_tokens=None)] + rows = [ + _make_atlas_row( + cell_id="cellA", + cache_mode="cold", + bench_backend="drive_load", + mean_input_tokens=None, + mean_output_tokens=None, + ) + ] assert methodology_problems(rows) == [] @@ -160,11 +169,18 @@ def test_methodology_descriptor_gate_exempts_failed_cell(): """A failed cell carries no measurement -> descriptor not required.""" rows = [ _make_atlas_row( - cell_id="dead", status="failed", - ttft_avg_ms=None, request_throughput_avg=None, - output_tps_per_user=None, output_tps_per_gpu=None, - cache_mode="unknown", dataset="unknown", cudagraph_mode="unknown", - kv_cache_dtype="unknown", image="unknown", gpu_memory_utilization=None, + cell_id="dead", + status="failed", + ttft_avg_ms=None, + request_throughput_avg=None, + output_tps_per_user=None, + output_tps_per_gpu=None, + cache_mode="unknown", + dataset="unknown", + cudagraph_mode="unknown", + kv_cache_dtype="unknown", + image="unknown", + gpu_memory_utilization=None, ) ] assert methodology_problems(rows) == [] @@ -175,8 +191,12 @@ def test_methodology_descriptor_gate_exempts_failed_cell(): def _status(sol_rigor: str) -> RenderStatusSummary: return RenderStatusSummary( - rendered=True, sol_complete=True, plot_ready_points=1, omitted_pages="", - dcgm_grounded=True, sol_rigor=sol_rigor, + rendered=True, + sol_complete=True, + plot_ready_points=1, + omitted_pages="", + dcgm_grounded=True, + sol_rigor=sol_rigor, ) @@ -203,8 +223,7 @@ def test_krhpa_l4_missing_block_flagged(tmp_path: Path): def test_krhpa_l4_exempt_reason_ok(tmp_path: Path): (tmp_path / "config.yaml").write_text( - "name: test\n" - "krhpa_exempt_reason: engine/config profiling, not a custom-kernel comparison\n" + "name: test\nkrhpa_exempt_reason: engine/config profiling, not a custom-kernel comparison\n" ) assert krhpa_problems(tmp_path, _status("L4")) == [] @@ -243,7 +262,10 @@ def test_publish_strict_refuses_unlabeled_cache_mode(tmp_path: Path): campaign_dir = _stage_campaign(tmp_path) # default rows: cache_mode unknown with pytest.raises(CampaignIncompleteError, match="cache_mode=unknown"): publish( - campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, + campaign_dir, + cfg=_stub_cfg(), + dry_run=True, + strict=True, s3_client_factory=lambda _cfg: _FakeS3Client(), ) @@ -253,7 +275,9 @@ def test_publish_no_strict_records_unlabeled_and_lands(tmp_path: Path): (strict=False) with the gap recorded (visible on atlas_v1.cache_mode).""" campaign_dir = _stage_campaign(tmp_path) result = publish( - campaign_dir, cfg=_stub_cfg(), dry_run=True, + campaign_dir, + cfg=_stub_cfg(), + dry_run=True, s3_client_factory=lambda _cfg: _FakeS3Client(), ) assert result.campaign.row_count == 1 @@ -262,13 +286,19 @@ def test_publish_no_strict_records_unlabeled_and_lands(tmp_path: Path): def test_publish_strict_passes_when_all_labeled(tmp_path: Path): """All rows cold + dcgm_grounded -> --strict publishes cleanly.""" campaign_dir = _stage_campaign(tmp_path) - _restage_atlas(campaign_dir, [ - _make_atlas_row(cell_id="cellA", cache_mode="cold"), - _make_atlas_row(cell_id="cellB", cache_mode="cold"), - ]) + _restage_atlas( + campaign_dir, + [ + _make_atlas_row(cell_id="cellA", cache_mode="cold"), + _make_atlas_row(cell_id="cellB", cache_mode="cold"), + ], + ) _write_status(campaign_dir, dcgm_grounded=True) # clear the hard DCGM gate publish( - campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, + campaign_dir, + cfg=_stub_cfg(), + dry_run=True, + strict=True, s3_client_factory=lambda _cfg: _FakeS3Client(), ) # must not raise @@ -281,7 +311,10 @@ def test_publish_strict_refuses_l4_without_krhpa(tmp_path: Path): (campaign_dir / "config.yaml").write_text("name: test\n") # no krhpa with pytest.raises(CampaignIncompleteError, match="krhpa"): publish( - campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, + campaign_dir, + cfg=_stub_cfg(), + dry_run=True, + strict=True, s3_client_factory=lambda _cfg: _FakeS3Client(), ) @@ -292,6 +325,9 @@ def test_publish_strict_passes_l4_with_krhpa(tmp_path: Path): _write_status(campaign_dir, sol_rigor="L4", dcgm_grounded=True) (campaign_dir / "config.yaml").write_text(_KRHPA_VALID) publish( - campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, + campaign_dir, + cfg=_stub_cfg(), + dry_run=True, + strict=True, s3_client_factory=lambda _cfg: _FakeS3Client(), ) # must not raise diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_model_eval.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_model_eval.py index 41020cc..0e3830d 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_model_eval.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_model_eval.py @@ -37,8 +37,7 @@ def test_import_model_eval_writes_quality_cell(tmp_path: Path): res.write_text(json.dumps(_RESULTS)) camp = tmp_path / "campaigns" / "glm51-eval-20260607T000000Z" camp.mkdir(parents=True) - out = import_model_eval(res, camp, model="GLM-5.1-NVFP4", hardware="GB300", - quant="NVFP4", tensor_parallel=4) + out = import_model_eval(res, camp, model="GLM-5.1-NVFP4", hardware="GB300", quant="NVFP4", tensor_parallel=4) assert out["n_metrics"] == 3 norm = json.loads((camp / "cells" / "model-eval" / "normalized.json").read_text()) assert isinstance(norm, list) and len(norm) == 1 @@ -66,9 +65,26 @@ def test_cli_import_model_eval(tmp_path: Path, capsys): camps = tmp_path / "campaigns" camp = camps / "glm51-eval-20260607T000000Z" camp.mkdir(parents=True) - rc = main(["import_model_eval", "--results", str(res), "--campaign", str(camp), - "--model", "GLM-5.1-NVFP4", "--hardware", "GB300", "--quant", "NVFP4", - "--tensor-parallel", "4", "--campaigns-dir", str(camps), "--json"]) + rc = main( + [ + "import_model_eval", + "--results", + str(res), + "--campaign", + str(camp), + "--model", + "GLM-5.1-NVFP4", + "--hardware", + "GB300", + "--quant", + "NVFP4", + "--tensor-parallel", + "4", + "--campaigns-dir", + str(camps), + "--json", + ] + ) assert rc == 0 env = json.loads(capsys.readouterr().out) assert env["n_metrics"] == 3 and env["cell_id"] == "model-eval" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_ncu_kernels.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_ncu_kernels.py index 0a3871b..e8ce351 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_ncu_kernels.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_ncu_kernels.py @@ -31,7 +31,6 @@ import_ncu_kernels, ) - # --------------------------------------------------------------------------- # Synthetic ncu CSV helpers # --------------------------------------------------------------------------- @@ -108,14 +107,18 @@ def _make_bundle( def test_positive_path(tmp_path): """S1: manifest declares ncu + paired sol+raw CSVs -> ncu_kernels.json.""" - sol = _sol_csv([ - ("multimem_all_reduce_kernel", 92.0, 8.5, 0.31, 8.0, 6.0, 4.0), - ("bmm_E2m1E2m1_Fp32_sm100f", 45.0, 88.0, 0.62, 8.0, 6.0, 8.0), - ]) - raw = _raw_csv([ - ("multimem_all_reduce_kernel", 1.2e9, 1.5e8, 8e7, 12340.0), - ("bmm_E2m1E2m1_Fp32_sm100f", 5e8, 8e9, 5e9, 23000.0), - ]) + sol = _sol_csv( + [ + ("multimem_all_reduce_kernel", 92.0, 8.5, 0.31, 8.0, 6.0, 4.0), + ("bmm_E2m1E2m1_Fp32_sm100f", 45.0, 88.0, 0.62, 8.0, 6.0, 8.0), + ] + ) + raw = _raw_csv( + [ + ("multimem_all_reduce_kernel", 1.2e9, 1.5e8, 8e7, 12340.0), + ("bmm_E2m1E2m1_Fp32_sm100f", 5e8, 8e9, 5e9, 23000.0), + ] + ) bundle = _make_bundle( tmp_path / "bundle", manifest=_VALID_MANIFEST, @@ -145,9 +148,7 @@ def test_positive_path(tmp_path): # Block limit factor: warps_limit (4.0) < shared (6.0) < regs (8.0) -> "warps" assert nccl["block_limit_factor"] == "warps" # Arithmetic intensity: flops/bytes - assert nccl["arithmetic_intensity_flops_per_byte"] == pytest.approx( - (1.5e8 + 8e7) / 1.2e9, rel=1e-6 - ) + assert nccl["arithmetic_intensity_flops_per_byte"] == pytest.approx((1.5e8 + 8e7) / 1.2e9, rel=1e-6) bmm = next(k for k in payload["kernels"] if k["name"] == "bmm_E2m1E2m1_Fp32_sm100f") assert bmm["category"] == "BMM-NVFP4" @@ -258,16 +259,20 @@ def test_multi_launch_aggregates_correctly(tmp_path): The importer averages SOL percentages and sums byte/flop counts. """ # Same kernel, 3 launches with slightly different numbers - sol = _sol_csv([ - ("k1", 90.0, 10.0, 0.30, 8, 6, 4), - ("k1", 92.0, 11.0, 0.32, 8, 6, 4), - ("k1", 94.0, 12.0, 0.34, 8, 6, 4), - ]) - raw = _raw_csv([ - ("k1", 1e9, 1e8, 1e8, 1000.0), - ("k1", 1e9, 1e8, 1e8, 1000.0), - ("k1", 1e9, 1e8, 1e8, 1000.0), - ]) + sol = _sol_csv( + [ + ("k1", 90.0, 10.0, 0.30, 8, 6, 4), + ("k1", 92.0, 11.0, 0.32, 8, 6, 4), + ("k1", 94.0, 12.0, 0.34, 8, 6, 4), + ] + ) + raw = _raw_csv( + [ + ("k1", 1e9, 1e8, 1e8, 1000.0), + ("k1", 1e9, 1e8, 1e8, 1000.0), + ("k1", 1e9, 1e8, 1e8, 1000.0), + ] + ) bundle = _make_bundle( tmp_path / "bundle", manifest=_VALID_MANIFEST, @@ -321,20 +326,12 @@ def _long_sol_csv(kernel: str, metrics: list[tuple[str, str, str]]) -> str: metrics: list of (metric_name, metric_unit, metric_value). Mirrors the real ``--page details --section SpeedOfLight`` export shape. """ - header = ( - '"ID","Process ID","Kernel Name","Section Name",' - '"Metric Name","Metric Unit","Metric Value","Rule Name"' - ) + header = '"ID","Process ID","Kernel Name","Section Name","Metric Name","Metric Unit","Metric Value","Rule Name"' lines = [header] for mname, munit, mval in metrics: - lines.append( - f'"0","942","{kernel}","GPU Speed Of Light Throughput",' - f'"{mname}","{munit}","{mval}",""' - ) + lines.append(f'"0","942","{kernel}","GPU Speed Of Light Throughput","{mname}","{munit}","{mval}",""') # ncu also emits a trailing rule row with empty Metric Name -> must be skipped. - lines.append( - f'"0","942","{kernel}","SpeedOfLight","","","","SOLBottleneck"' - ) + lines.append(f'"0","942","{kernel}","SpeedOfLight","","","","SOLBottleneck"') return "\n".join(lines) + "\n" @@ -359,10 +356,7 @@ def test_long_format_sol_only_basic_set(tmp_path): ) # Basic-set raw page: no dram__bytes / sm__sass_thread_inst_executed_op # columns; carries the occupancy proxy instead. - raw = ( - '"ID","Kernel Name","sm__warps_active.avg.pct_of_peak_sustained_active"\n' - '"0","triton_red_fused_2","91.13"\n' - ) + raw = '"ID","Kernel Name","sm__warps_active.avg.pct_of_peak_sustained_active"\n"0","triton_red_fused_2","91.13"\n' bundle = _make_bundle( tmp_path / "bundle", manifest=_VALID_MANIFEST, @@ -401,12 +395,12 @@ def test_ncu_2026_1_1_units_row_filtered(tmp_path): sol_with_units_row = ( _SOL_HEADER + "\n" '"","","%","%","","","",""\n' # units-row (Kernel Name empty, "%" units) - + '"0","triton_red_fused","45.0","88.0","0.62","8.0","6.0","8.0"\n' + + '"0","triton_red_fused","45.0","88.0","0.62","8.0","6.0","8.0"\n' ) raw_with_units_row = ( _RAW_HEADER + "\n" '"","","byte","inst","inst","ns"\n' # units-row - + '"0","triton_red_fused","5e8","8e9","5e9","23000.0"\n' + + '"0","triton_red_fused","5e8","8e9","5e9","23000.0"\n' ) bundle = _make_bundle( tmp_path / "bundle", @@ -434,15 +428,11 @@ def test_dram_bytes_mbyte_unit_scaled_to_bytes(tmp_path): Regression for the unit-unaware sum that produced ~1e6-too-small bytes (the published dram_bytes_total=11.64 / 90 artifacts). """ - sol = ( - _SOL_HEADER + "\n" - '"","","%","%","","","",""\n' - + '"0","fmha_test","48.6","24.6","0.23","8.0","6.0","8.0"\n' - ) + sol = _SOL_HEADER + '\n"","","%","%","","","",""\n' + '"0","fmha_test","48.6","24.6","0.23","8.0","6.0","8.0"\n' raw = ( _RAW_HEADER + "\n" '"","","Mbyte","inst","inst","usecond"\n' # units row: bytes in Mbyte - + '"0","fmha_test","248.4","1000","2000","66.6"\n' + + '"0","fmha_test","248.4","1000","2000","66.6"\n' ) bundle = _make_bundle( tmp_path / "bundle", @@ -456,9 +446,7 @@ def test_dram_bytes_mbyte_unit_scaled_to_bytes(tmp_path): # 248.4 Mbyte -> 248.4e6 bytes (NOT 248.4 raw). assert k["dram_bytes_total"] == pytest.approx(248.4e6) # AI denominator uses the scaled bytes. - assert k["arithmetic_intensity_flops_per_byte"] == pytest.approx( - 3000.0 / 248.4e6, rel=1e-6 - ) + assert k["arithmetic_intensity_flops_per_byte"] == pytest.approx(3000.0 / 248.4e6, rel=1e-6) def test_tensor_core_flops_folded_into_ai(tmp_path): @@ -493,9 +481,7 @@ def test_tensor_core_flops_folded_into_ai(tmp_path): # scalar stays separate (CUDA-core ffma only). assert k["sm_flops_total"] == pytest.approx(100.0) # AI uses scalar + tensor over bytes -- dominated by tensor, NOT 100/1e6. - assert k["arithmetic_intensity_flops_per_byte"] == pytest.approx( - (5e6 + 100.0) / 1e6, rel=1e-6 - ) + assert k["arithmetic_intensity_flops_per_byte"] == pytest.approx((5e6 + 100.0) / 1e6, rel=1e-6) # achieved TFLOPS computed off the total (non-null, tensor-inclusive). assert k["achieved_tflops"] is not None and k["achieved_tflops"] > 0 @@ -506,10 +492,7 @@ def test_raw_time_usecond_unit_scaled_for_tflops(tmp_path): to ns -- else achieved_tflops is ~1000x inflated. (Real-capture regression.) """ # No SoL Duration column -> rate falls back to raw time. - sol = ( - '"ID","Kernel Name","DRAM Throughput [%]","Compute (SM) Throughput [%]"\n' - '"0","k_t","48.0","25.0"\n' - ) + sol = '"ID","Kernel Name","DRAM Throughput [%]","Compute (SM) Throughput [%]"\n"0","k_t","48.0","25.0"\n' # 1 GFLOP over 1000 usecond (= 1e-3 s) -> 1e9 / 1e-3 / 1e12 = 1.0 TFLOPS # WHEN the usecond unit is scaled to ns. Unscaled (treating 1000 as ns) it # would be ~1000 TFLOPS -- so 1.0 proves the unit scaling fired. diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_path_forward.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_path_forward.py index 50552dc..1641771 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_path_forward.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_path_forward.py @@ -87,9 +87,7 @@ def test_next_lever_problems_present_is_empty(tmp_path: Path): def test_next_lever_problems_frontier_exhausted_is_allowed(tmp_path: Path): """A genuinely-maxed dimension opts out via the non-empty escape value.""" - (tmp_path / "config.yaml").write_text( - "name: t\nnext_lever: 'frontier-exhausted: HBM SoL 94% at c=256'\n" - ) + (tmp_path / "config.yaml").write_text("name: t\nnext_lever: 'frontier-exhausted: HBM SoL 94% at c=256'\n") assert next_lever_problems(tmp_path) == [] @@ -116,7 +114,10 @@ def test_publish_strict_refuses_without_next_lever(tmp_path: Path): _strip_next_lever(cd) with pytest.raises(CampaignIncompleteError, match="next_lever"): publish( - cd, cfg=_stub_cfg(), dry_run=True, strict=True, + cd, + cfg=_stub_cfg(), + dry_run=True, + strict=True, s3_client_factory=lambda _cfg: _FakeS3Client(), ) @@ -127,7 +128,9 @@ def test_publish_no_strict_records_missing_next_lever_and_lands(tmp_path: Path): cd = _stage_campaign(tmp_path) _strip_next_lever(cd) result = publish( - cd, cfg=_stub_cfg(), dry_run=True, + cd, + cfg=_stub_cfg(), + dry_run=True, s3_client_factory=lambda _cfg: _FakeS3Client(), ) assert result.campaign.row_count == 1 diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_perf_report.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_perf_report.py index 7992cbf..d4fa007 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_perf_report.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_perf_report.py @@ -21,9 +21,7 @@ import io import json -import shutil import sys -import tempfile from collections import defaultdict from pathlib import Path @@ -33,6 +31,8 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) +from datetime import UTC + from tools.perf_tune_report.aggregator import aggregate from tools.perf_tune_report.capture_signature import variant_key_for from tools.perf_tune_report.coverage import summarize @@ -57,16 +57,13 @@ normalize_outputs as normalize_outputs_vllm_sweep, ) from tools.perf_tune_report.schema import ( - STATUS_EVICTED, STATUS_FAILED, STATUS_FULL, - STATUS_PARTIAL, AtlasCell, read_jsonl, write_jsonl, ) - FIXTURE_PATH = Path(__file__).parent / "fixtures" / "synthetic_atlas.jsonl" @@ -74,36 +71,66 @@ # schema.py # --------------------------------------------------------------------------- + def test_schema_rejects_unknown_status(): with pytest.raises(ValueError, match="status must be one of"): AtlasCell( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status="bogus", + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status="bogus", ) def test_schema_rejects_unknown_parallel_strategy(): with pytest.raises(ValueError, match="parallel_strategy"): AtlasCell( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="PP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status=STATUS_FULL, + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="PP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status=STATUS_FULL, ) def test_schema_has_metrics_property(): cell = AtlasCell( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status=STATUS_FULL, - ttft_avg_ms=100.0, request_throughput_avg=0.1, + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status=STATUS_FULL, + ttft_avg_ms=100.0, + request_throughput_avg=0.1, ) assert cell.has_metrics is True cell_no_metrics = AtlasCell( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=1024, concurrency=0, status=STATUS_FAILED, + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=0, + status=STATUS_FAILED, ) assert cell_no_metrics.has_metrics is False @@ -136,6 +163,7 @@ def test_schema_jsonl_skips_blank_and_comment_lines(): # coverage.py # --------------------------------------------------------------------------- + def test_coverage_matches_pdf_header(): rows = read_jsonl(FIXTURE_PATH) summary = summarize(rows) @@ -159,11 +187,20 @@ def test_coverage_header_line_format(): def test_coverage_note_line_omitted_when_no_evicted(): rows = [ AtlasCell( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status=STATUS_FULL, - ttft_avg_ms=1.0, request_throughput_avg=0.1, - output_tps_per_user=1.0, output_tps_per_gpu=1.0, + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status=STATUS_FULL, + ttft_avg_ms=1.0, + request_throughput_avg=0.1, + output_tps_per_user=1.0, + output_tps_per_gpu=1.0, ), ] summary = summarize(rows) @@ -172,12 +209,20 @@ def test_coverage_note_line_omitted_when_no_evicted(): def _full_cell(cell_id: str, *, plottable: bool) -> AtlasCell: return AtlasCell( - cell_id=cell_id, model="m", hardware="B200", quant="NVFP4", - tensor_parallel=8, parallel_strategy="TP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status=STATUS_FULL, + cell_id=cell_id, + model="m", + hardware="B200", + quant="NVFP4", + tensor_parallel=8, + parallel_strategy="TP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status=STATUS_FULL, ttft_avg_ms=1.0 if plottable else None, request_throughput_avg=0.1 if plottable else None, - output_tps_per_user=1.0, output_tps_per_gpu=1.0, + output_tps_per_user=1.0, + output_tps_per_gpu=1.0, ) @@ -248,24 +293,33 @@ def test_renderer_zero_plot_ready_records_empty_scatter(tmp_path): def _row_with_capture(captured_at: str) -> AtlasCell: return AtlasCell( - cell_id="ctrl-nomtp", model="GLM-5.1-NVFP4", hardware="B200", quant="NVFP4", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=512, concurrency=1, status=STATUS_FULL, - ttft_avg_ms=1.0, request_throughput_avg=0.1, - output_tps_per_user=1.0, output_tps_per_gpu=1.0, + cell_id="ctrl-nomtp", + model="GLM-5.1-NVFP4", + hardware="B200", + quant="NVFP4", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=512, + concurrency=1, + status=STATUS_FULL, + ttft_avg_ms=1.0, + request_throughput_avg=0.1, + output_tps_per_user=1.0, + output_tps_per_gpu=1.0, captured_at=captured_at, ) def test_pdf_provenance_embeds_utc_runid_and_window(): - from datetime import datetime, timezone + from datetime import datetime from tools.perf_tune_report.renderer.render_report import build_pdf_provenance # A local-clock instant a calendar day BEHIND the UTC instant (22:41 PDT on # 05-30 == 05:41Z on 05-31) is exactly the off-by-a-day confusion this # provenance stamp removes -- the stamp is computed in UTC. - rendered_at = datetime(2026, 5, 31, 5, 41, 0, tzinfo=timezone.utc) + rendered_at = datetime(2026, 5, 31, 5, 41, 0, tzinfo=UTC) rid = "20260531T081514Z-glm51-deepep-shapes-5k1k" prov = build_pdf_provenance(rid, [_row_with_capture("2026-05-31T05:43:09Z")], rendered_at) @@ -282,7 +336,7 @@ def test_pdf_provenance_embeds_utc_runid_and_window(): def test_pdf_provenance_brackets_multiple_capture_windows(): - from datetime import datetime, timezone + from datetime import datetime from tools.perf_tune_report.renderer.render_report import build_pdf_provenance @@ -291,16 +345,16 @@ def test_pdf_provenance_brackets_multiple_capture_windows(): _row_with_capture("2026-05-31T08:12:00Z"), _row_with_capture("2026-05-31T05:43:09Z"), # duplicate collapses ] - prov = build_pdf_provenance("rid", rows, datetime.now(timezone.utc)) + prov = build_pdf_provenance("rid", rows, datetime.now(UTC)) assert prov["bench_window"] == "2026-05-31T05:43:09Z .. 2026-05-31T08:12:00Z" def test_pdf_provenance_handles_missing_capture(): - from datetime import datetime, timezone + from datetime import datetime from tools.perf_tune_report.renderer.render_report import build_pdf_provenance - prov = build_pdf_provenance("rid", [_row_with_capture("")], datetime.now(timezone.utc)) + prov = build_pdf_provenance("rid", [_row_with_capture("")], datetime.now(UTC)) assert prov["bench_window"] == "unknown" @@ -308,21 +362,40 @@ def test_pdf_provenance_handles_missing_capture(): # CONTRACT shape # --------------------------------------------------------------------------- + def test_contract_verb_set(): """The perf_tune_report CONTRACT verb set, including champion_select and import_variant_ab.""" assert set(CONTRACT) == { - "campaign_init", "cell_run", "atlas_aggregate", - "report_render", "report_smoke", "publish_to_lake", - "import_perf_bench", "campaign_run", - "kernel_profile", "graph_diff", - "raw_bench_compare", "import_ncu", "import_nsys", "dcgm_correlate", - "experiments_index", "tpm_summary", "value_view", - "import_roofline_sweep", "fleet_leaderboard", - "champion_select", "import_variant_ab", - "capture_plan", "materialize_capture_reuse", - "experiment_inventory", "portability_view", "import_model_eval", "trend_view", - "kernel_reproducer_scaffold", "import_workloads", + "campaign_init", + "cell_run", + "atlas_aggregate", + "report_render", + "report_smoke", + "publish_to_lake", + "import_perf_bench", + "campaign_run", + "kernel_profile", + "graph_diff", + "raw_bench_compare", + "import_ncu", + "import_nsys", + "dcgm_correlate", + "experiments_index", + "tpm_summary", + "value_view", + "import_roofline_sweep", + "fleet_leaderboard", + "champion_select", + "import_variant_ab", + "capture_plan", + "materialize_capture_reuse", + "experiment_inventory", + "portability_view", + "import_model_eval", + "trend_view", + "kernel_reproducer_scaffold", + "import_workloads", } @@ -388,8 +461,11 @@ def test_contract_publish_to_lake_is_external_and_ack_gated(): def test_contract_writes_artifacts_verbs(): for verb in ( - "campaign_init", "atlas_aggregate", "report_render", - "import_perf_bench", "graph_diff", + "campaign_init", + "atlas_aggregate", + "report_render", + "import_perf_bench", + "graph_diff", ): assert CONTRACT[verb]["safety"] == "writes_artifacts" assert CONTRACT[verb]["ack"] is None @@ -399,18 +475,27 @@ def test_contract_writes_artifacts_verbs(): # CLI ack-gating # --------------------------------------------------------------------------- + def _make_campaign(tmp_path: Path, monkeypatch) -> Path: import yaml + monkeypatch.setenv("PERFREPORT_CAMPAIGNS_DIR", str(tmp_path)) cfg = { "name": "test", - "cells": [{ - "cell_id": "cell1", - "model": "X", "hardware": "H100", "quant": "FP8", - "tensor_parallel": 8, "parallel_strategy": "EP", "mtp": False, - "max_num_batched_tokens": 1024, "concurrencies": [1, 8], - "vllm_sweep": {"serve_cmd": "vllm serve X", "bench_cmd": "vllm bench serve --model X"}, - }], + "cells": [ + { + "cell_id": "cell1", + "model": "X", + "hardware": "H100", + "quant": "FP8", + "tensor_parallel": 8, + "parallel_strategy": "EP", + "mtp": False, + "max_num_batched_tokens": 1024, + "concurrencies": [1, 8], + "vllm_sweep": {"serve_cmd": "vllm serve X", "bench_cmd": "vllm bench serve --model X"}, + } + ], } cfg_path = tmp_path / "test.yaml" cfg_path.write_text(yaml.safe_dump(cfg)) @@ -594,10 +679,18 @@ def test_kernel_importers_reject_cell_symlink_escape( def test_cli_cell_run_refuses_without_ack(tmp_path, monkeypatch, capsys): _make_campaign(tmp_path, monkeypatch) - rc = main([ - "cell_run", "--campaign", "test", "--cell", "cell1", - "--backend", "vllm-sweep", "--json", - ]) + rc = main( + [ + "cell_run", + "--campaign", + "test", + "--cell", + "cell1", + "--backend", + "vllm-sweep", + "--json", + ] + ) captured = capsys.readouterr() assert rc == 2 assert "ack-gated" in captured.err @@ -605,10 +698,19 @@ def test_cli_cell_run_refuses_without_ack(tmp_path, monkeypatch, capsys): def test_cli_cell_run_dry_run_does_not_require_ack(tmp_path, monkeypatch): _make_campaign(tmp_path, monkeypatch) - rc = main([ - "cell_run", "--campaign", "test", "--cell", "cell1", - "--backend", "vllm-sweep", "--dry-run", "--json", - ]) + rc = main( + [ + "cell_run", + "--campaign", + "test", + "--cell", + "cell1", + "--backend", + "vllm-sweep", + "--dry-run", + "--json", + ] + ) assert rc == 0 @@ -620,13 +722,17 @@ def test_campaign_run_dry_run_accepts_campaign_init_config( campaign = _make_campaign(tmp_path, monkeypatch) capsys.readouterr() - rc = main([ - "campaign_run", - "--config", str(tmp_path / "test.yaml"), - "--campaign", str(campaign), - "--dry-run", - "--json", - ]) + rc = main( + [ + "campaign_run", + "--config", + str(tmp_path / "test.yaml"), + "--campaign", + str(campaign), + "--dry-run", + "--json", + ] + ) assert rc == 0 payload = json.loads(capsys.readouterr().out) @@ -649,26 +755,37 @@ def test_campaign_run_endpoint_only_needs_no_fake_helm_config( config = { "name": "endpoint-only", "focus": "latency", - "cells": [{ - "cell_id": "endpoint-cell", - "backend": "aa", - "model": "example/model", - "hardware": "B200", - "quant": "FP8", - "tensor_parallel": 4, - "parallel_strategy": "TP", - "mtp": False, - "max_num_batched_tokens": 2048, - "concurrencies": [1], - "aa": {"model": "example/model", "url": "http://example", "shape": "aa-1k"}, - }], + "cells": [ + { + "cell_id": "endpoint-cell", + "backend": "aa", + "model": "example/model", + "hardware": "B200", + "quant": "FP8", + "tensor_parallel": 4, + "parallel_strategy": "TP", + "mtp": False, + "max_num_batched_tokens": 2048, + "concurrencies": [1], + "aa": {"model": "example/model", "url": "http://example", "shape": "aa-1k"}, + } + ], } config_path = tmp_path / "endpoint.yaml" config_path.write_text(yaml.safe_dump(config)) - assert main([ - "campaign_init", "--config", str(config_path), - "--slug", "endpoint", "--json", - ]) == 0 + assert ( + main( + [ + "campaign_init", + "--config", + str(config_path), + "--slug", + "endpoint", + "--json", + ] + ) + == 0 + ) campaign = next(tmp_path.glob("*-endpoint")) capsys.readouterr() helm_calls: list[object] = [] @@ -690,13 +807,17 @@ def test_campaign_run_endpoint_only_needs_no_fake_helm_config( lambda: step_fns, ) - rc = main([ - "campaign_run", - "--config", str(config_path), - "--campaign", str(campaign), - "--i-understand-this-mutates-cluster", - "--json", - ]) + rc = main( + [ + "campaign_run", + "--config", + str(config_path), + "--campaign", + str(campaign), + "--i-understand-this-mutates-cluster", + "--json", + ] + ) assert rc == 0 assert helm_calls == [] @@ -719,6 +840,7 @@ def test_campaign_init_copies_provenance_from_bundle(tmp_path, monkeypatch): campaign: provenance.json sidecar + config.yaml provenance: + SOURCE.md source bullets (so it flows to the lake's campaign_v1 source columns).""" import yaml + monkeypatch.setenv("PERFREPORT_CAMPAIGNS_DIR", str(tmp_path / "campaigns")) bundle = tmp_path / "bundle" bundle.mkdir() @@ -739,11 +861,18 @@ def test_campaign_init_copies_provenance_from_bundle(tmp_path, monkeypatch): ) cfg_path = tmp_path / "x.yaml" cfg_path.write_text(yaml.safe_dump({"name": "x", "cells": []})) - rc = main([ - "campaign_init", "--config", str(cfg_path), - "--experiment-id", "glm51-x-20260604T103817Z", - "--evidence-bundle", str(bundle), "--json", - ]) + rc = main( + [ + "campaign_init", + "--config", + str(cfg_path), + "--experiment-id", + "glm51-x-20260604T103817Z", + "--evidence-bundle", + str(bundle), + "--json", + ] + ) assert rc == 0 camp = tmp_path / "campaigns" / "glm51-x-20260604T103817Z" assert (camp / "provenance.json").is_file() @@ -758,6 +887,7 @@ def test_campaign_init_copies_provenance_from_bundle(tmp_path, monkeypatch): # Aggregator # --------------------------------------------------------------------------- + def _seed_campaign_from_fixture(tmp_path: Path) -> Path: """Build a campaign dir from the bundled synthetic fixture.""" campaign_dir = tmp_path / "seeded" @@ -793,9 +923,7 @@ def test_aggregator_raises_on_schema_drift(tmp_path): campaign_dir = tmp_path / "drift" cell_dir = campaign_dir / "cells" / "bad" cell_dir.mkdir(parents=True) - (cell_dir / "normalized.json").write_text( - json.dumps([{"cell_id": "bad", "status": "weird-status"}]) - ) + (cell_dir / "normalized.json").write_text(json.dumps([{"cell_id": "bad", "status": "weird-status"}])) with pytest.raises(ValueError, match="schema"): aggregate(campaign_dir) @@ -805,6 +933,7 @@ def test_aggregator_raises_on_schema_drift(tmp_path): # representative backend output rather than only the dry-run command path). # --------------------------------------------------------------------------- + def _make_test_cell(concurrencies=(1, 8, 32)) -> CellConfig: return CellConfig( cell_id="test-cell", @@ -832,12 +961,16 @@ def test_runner_vllm_sweep_normalize_full_status_and_derived_metrics(tmp_path): raw_dir = tmp_path / "raw" raw_dir.mkdir() for c in cell.concurrencies: - (raw_dir / f"c{c}.json").write_text(json.dumps({ - "max_concurrency": c, - "median_ttft_ms": 100.0 * c, - "request_throughput": 0.1 * c, - "output_throughput": 800.0 * c, - })) + (raw_dir / f"c{c}.json").write_text( + json.dumps( + { + "max_concurrency": c, + "median_ttft_ms": 100.0 * c, + "request_throughput": 0.1 * c, + "output_throughput": 800.0 * c, + } + ) + ) cell_dir = tmp_path / "cells" / cell.cell_id cell_dir.mkdir(parents=True) @@ -889,12 +1022,16 @@ def _normalize_one(cell: CellConfig, tmp_path): work = tmp_path / f"work-{cell.cell_id}-{_normalize_one.n}" raw_dir = work / "raw" raw_dir.mkdir(parents=True) - (raw_dir / "c1.json").write_text(json.dumps({ - "max_concurrency": 1, - "median_ttft_ms": 50.0, - "request_throughput": 1.0, - "output_throughput": 60.0, - })) + (raw_dir / "c1.json").write_text( + json.dumps( + { + "max_concurrency": 1, + "median_ttft_ms": 50.0, + "request_throughput": 1.0, + "output_throughput": 60.0, + } + ) + ) cell_dir = work / "cells" / cell.cell_id cell_dir.mkdir(parents=True) rows, _ = normalize_outputs_vllm_sweep(cell, raw_dir, cell_dir) @@ -919,9 +1056,7 @@ def test_runner_vllm_sweep_variant_key_distinguishes_mtp_k(tmp_path): row_k3 = _normalize_one(_spec_cell(3), tmp_path) assert variant_key_for(row_k2) != variant_key_for(row_k3) # And it is image-INDEPENDENT: same K + same knobs, different image == same key. - row_k3_newimg = _normalize_one( - _spec_cell(3, extras_overrides={"image": "infr/vllm:v9.9.9"}), tmp_path - ) + row_k3_newimg = _normalize_one(_spec_cell(3, extras_overrides={"image": "infr/vllm:v9.9.9"}), tmp_path) assert variant_key_for(row_k3) == variant_key_for(row_k3_newimg) @@ -936,29 +1071,41 @@ def test_runner_aiperf_normalize_extracts_nested_metrics(tmp_path): # c1: metrics at the TOP level using primary field names. (raw_dir / "c1").mkdir(parents=True) - (raw_dir / "c1" / "profile_export_aiperf.json").write_text(json.dumps({ - "median_ttft_ms": 50.0, - "request_throughput": 0.5, - "output_throughput": 1000.0, - })) + (raw_dir / "c1" / "profile_export_aiperf.json").write_text( + json.dumps( + { + "median_ttft_ms": 50.0, + "request_throughput": 0.5, + "output_throughput": 1000.0, + } + ) + ) # c8: metrics nested under `results` using SECONDARY field aliases. (raw_dir / "c8").mkdir(parents=True) - (raw_dir / "c8" / "profile_export_aiperf.json").write_text(json.dumps({ - "results": { - "mean_ttft_ms": 150.0, - "request_throughput_avg": 1.5, - "output_throughput_tps": 8000.0, - } - })) + (raw_dir / "c8" / "profile_export_aiperf.json").write_text( + json.dumps( + { + "results": { + "mean_ttft_ms": 150.0, + "request_throughput_avg": 1.5, + "output_throughput_tps": 8000.0, + } + } + ) + ) # c32: metrics nested under `summary` using TERTIARY field aliases. (raw_dir / "c32").mkdir(parents=True) - (raw_dir / "c32" / "profile_export_aiperf.json").write_text(json.dumps({ - "summary": { - "ttft_avg_ms": 300.0, - "request_throughput": 3.0, - "output_tps": 24000.0, - } - })) + (raw_dir / "c32" / "profile_export_aiperf.json").write_text( + json.dumps( + { + "summary": { + "ttft_avg_ms": 300.0, + "request_throughput": 3.0, + "output_tps": 24000.0, + } + } + ) + ) cell_dir = tmp_path / "cells" / cell.cell_id cell_dir.mkdir(parents=True) @@ -991,9 +1138,7 @@ def test_runner_normalize_no_data_yields_failed(tmp_path): # aiperf: raw_dir with a JSON file missing all metric fields. aiperf_raw = tmp_path / "aiperf-raw" (aiperf_raw / "c1").mkdir(parents=True) - (aiperf_raw / "c1" / "profile_export_aiperf.json").write_text( - json.dumps({"unrelated": "data"}) - ) + (aiperf_raw / "c1" / "profile_export_aiperf.json").write_text(json.dumps({"unrelated": "data"})) rows_aiperf, status_aiperf = normalize_outputs_aiperf(cell, aiperf_raw, cell_dir) assert status_aiperf == STATUS_FAILED assert rows_aiperf == [] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_portability_view.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_portability_view.py index 0add520..8f8272b 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_portability_view.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_portability_view.py @@ -25,13 +25,33 @@ def _registry() -> dict: return { "baseline_context": "x", "findings": [ - {"id": "a", "title": "A", "lifecycle": "done", "applies_to": "all", - "validated_on": ["GLM-5.1"], "candidate_on": ["Kimi-K2.6", "DSv3.2"], - "refuted_on": []}, - {"id": "b", "title": "B", "lifecycle": "closed_negative", "applies_to": "moe", - "validated_on": [], "candidate_on": [], "refuted_on": ["GLM-5.1"]}, - {"id": "c", "title": "C", "lifecycle": "not_done", "applies_to": "various", - "validated_on": [], "candidate_on": [], "refuted_on": []}, # names no model + { + "id": "a", + "title": "A", + "lifecycle": "done", + "applies_to": "all", + "validated_on": ["GLM-5.1"], + "candidate_on": ["Kimi-K2.6", "DSv3.2"], + "refuted_on": [], + }, + { + "id": "b", + "title": "B", + "lifecycle": "closed_negative", + "applies_to": "moe", + "validated_on": [], + "candidate_on": [], + "refuted_on": ["GLM-5.1"], + }, + { + "id": "c", + "title": "C", + "lifecycle": "not_done", + "applies_to": "various", + "validated_on": [], + "candidate_on": [], + "refuted_on": [], + }, # names no model ], } @@ -78,8 +98,9 @@ def test_cli_portability_view(tmp_path, capsys): campaigns = tmp_path / "campaigns" campaigns.mkdir() out = tmp_path / "PORTABILITY-MATRIX.md" - rc = main(["portability_view", "--registry", str(reg), - "--campaigns-dir", str(campaigns), "--out", str(out), "--json"]) + rc = main( + ["portability_view", "--registry", str(reg), "--campaigns-dir", str(campaigns), "--out", str(out), "--json"] + ) assert rc == 0 env = json.loads(capsys.readouterr().out) assert env["lever_count"] == 2 diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_production_steps.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_production_steps.py index 6cdb072..efea765 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_production_steps.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_production_steps.py @@ -3,29 +3,27 @@ from __future__ import annotations import json -import subprocess from pathlib import Path from typing import Any import pytest from tools.perf_tune_report.orchestrator import production_step_fns +from tools.perf_tune_report.orchestrator.campaign_run import CellPlan, StepFns from tools.perf_tune_report.orchestrator.production_steps import ( _run_cmd, step_aggregate, step_baseline_diff, step_baseline_record, step_bench, + step_dcgm_correlate, step_drain, step_helm_upgrade, step_import, step_render, step_resume, - step_warmup, step_zymtrace, ) -from tools.perf_tune_report.orchestrator.campaign_run import CellPlan, StepFns - # --------------------------------------------------------------------------- # 1. production_step_fns() returns a fully populated StepFns @@ -104,9 +102,7 @@ def fake(cmd: list[str], timeout: int = 120) -> bool: calls.append(cmd) return True - monkeypatch.setattr( - "tools.perf_tune_report.orchestrator.production_steps._run_cmd", fake - ) + monkeypatch.setattr("tools.perf_tune_report.orchestrator.production_steps._run_cmd", fake) assert step_drain(["node-1", "node-2"], "cell-x") is True assert calls == [ ["kubectl", "cordon", "node-1"], @@ -126,9 +122,7 @@ def fake(cmd: list[str], timeout: int = 120) -> bool: captured.append(cmd) return True - monkeypatch.setattr( - "tools.perf_tune_report.orchestrator.production_steps._run_cmd", fake - ) + monkeypatch.setattr("tools.perf_tune_report.orchestrator.production_steps._run_cmd", fake) ok = step_helm_upgrade( namespace="inference", release="basic-inference", @@ -153,9 +147,7 @@ def fake(cmd: list[str], timeout: int = 120) -> bool: captured.append(cmd) return True - monkeypatch.setattr( - "tools.perf_tune_report.orchestrator.production_steps._run_cmd", fake - ) + monkeypatch.setattr("tools.perf_tune_report.orchestrator.production_steps._run_cmd", fake) step_helm_upgrade( namespace="inference", release="basic-inference", @@ -231,7 +223,9 @@ def fake_run_cell(config, campaign_dir, **kwargs): def test_step_bench_aiperf_returns_false() -> None: """aiperf backend is intentionally a no-op in production_steps.""" cell = CellPlan( - id="test-cell", backend="aiperf", concurrencies=(8,), + id="test-cell", + backend="aiperf", + concurrencies=(8,), ) assert step_bench(Path("/tmp"), cell) is False @@ -239,7 +233,9 @@ def test_step_bench_aiperf_returns_false() -> None: def test_step_bench_trtllm_returns_false() -> None: """trtllm backend is a stub - run_cell raises NotImplementedError.""" cell = CellPlan( - id="test-cell", backend="trtllm", concurrencies=(8,), + id="test-cell", + backend="trtllm", + concurrencies=(8,), ) assert step_bench(Path("/tmp"), cell) is False @@ -292,6 +288,20 @@ def test_step_render_returns_false_when_no_atlas(tmp_path: Path) -> None: assert step_render(campaign, tmp_path / "out.pdf") is False +def test_step_dcgm_correlate_returns_false_for_malformed_frozen_yaml( + tmp_path: Path, +) -> None: + campaign = tmp_path / "campaign" + cell = campaign / "cells" / "c1" + cell.mkdir(parents=True) + (cell / "dcgm-frozen.yaml").write_text("[]\n") + ceilings = tmp_path / "perf-tune-report" / "configs" / "sol-ceilings.yaml" + ceilings.parent.mkdir(parents=True) + ceilings.write_text("{}\n") + + assert step_dcgm_correlate(campaign, "c1") is False + + # --------------------------------------------------------------------------- # 8. production baseline wiring uses one directional scalar headline # --------------------------------------------------------------------------- @@ -333,12 +343,21 @@ def _write_headline_campaign( } for concurrency, value in zip((1, 2), values, strict=True) ] - (campaign / "atlas.jsonl").write_text( - "".join(json.dumps(row) + "\n" for row in rows) - ) + (campaign / "atlas.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) return campaign +@pytest.mark.parametrize("payload_name", ["kernels.json", "dcgm_correlation.json"]) +def test_step_render_returns_false_for_malformed_domain_payload( + tmp_path: Path, + payload_name: str, +) -> None: + campaign = _write_headline_campaign(tmp_path, (80.0, 100.0)) + (campaign / "cells" / "c1" / payload_name).write_text("{bad") + + assert step_render(campaign, tmp_path / "out.pdf") is False + + def test_step_baseline_record_extracts_supported_scalar_headline( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -348,19 +367,8 @@ def test_step_baseline_record_extracts_supported_scalar_headline( assert step_baseline_record(campaign, "c1") is True - measurement_dir = ( - root - / "experiments" - / "artifacts" - / "perf-baselines" - / "inference" - / "output_tps_per_gpu-c2" - ) - entries = [ - entry - for entry in measurement_dir.iterdir() - if entry.is_dir() - ] + measurement_dir = root / "experiments" / "artifacts" / "perf-baselines" / "inference" / "output_tps_per_gpu-c2" + entries = [entry for entry in measurement_dir.iterdir() if entry.is_dir()] assert len(entries) == 1 baseline = json.loads((entries[0] / "baseline.json").read_text()) assert baseline["value"] == 100.0 @@ -384,17 +392,8 @@ def test_step_baseline_diff_parses_real_directional_verdict( root = _seed_baseline_repo(tmp_path, monkeypatch) campaign = _write_headline_campaign(tmp_path, (80.0, 100.0)) assert step_baseline_record(campaign, "c1") is True - measurement_dir = ( - root - / "experiments" - / "artifacts" - / "perf-baselines" - / "inference" - / "output_tps_per_gpu-c2" - ) - baseline_dir = next( - entry for entry in measurement_dir.iterdir() if entry.is_dir() - ) + measurement_dir = root / "experiments" / "artifacts" / "perf-baselines" / "inference" / "output_tps_per_gpu-c2" + baseline_dir = next(entry for entry in measurement_dir.iterdir() if entry.is_dir()) _write_headline_campaign(tmp_path, candidate) assert step_baseline_diff(campaign, "c1", str(baseline_dir)) == expected diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_provenance.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_provenance.py index c5f4fef..4dc0dae 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_provenance.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_provenance.py @@ -3,6 +3,7 @@ from __future__ import annotations import pytest +import yaml from tools.perf_tune_report import provenance as p @@ -143,12 +144,13 @@ def test_render_block_roundtrips(): def test_malformed_block_raises(): bad = "```provenance\n: : : not yaml : :\n- [unterminated\n```\n" - with pytest.raises(Exception): + with pytest.raises(yaml.YAMLError): p.parse_text(bad) # --- provenance_match_problems (code-under-test provenance match, rigor principle p) --- + def test_match_ok_same_delivery_and_commit(): refs = [{"delivery": "infr-patch", "commit": "abc123def456"}] camp = {"delivery": "infr-patch", "vllm_commit": "abc123def456789"} # flattened, full SHA @@ -191,6 +193,4 @@ def test_match_no_false_positive_on_missing_info(): assert p.provenance_match_problems([{"delivery": "infr-patch"}], None) == [] assert p.provenance_match_problems([{"delivery": "infr-patch"}], {}) == [] # campaign provenance carries neither delivery nor commit -> nothing to assert. - assert p.provenance_match_problems( - [{"delivery": "infr-patch", "commit": "x"}], {"foo": "bar"} - ) == [] + assert p.provenance_match_problems([{"delivery": "infr-patch", "commit": "x"}], {"foo": "bar"}) == [] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_publish_to_lake.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_publish_to_lake.py index 0fae246..908d8d4 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_publish_to_lake.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_publish_to_lake.py @@ -16,7 +16,7 @@ import json import sys -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -28,25 +28,25 @@ from tools.perf_tune_report.lake_writer import ( ATLAS_TABLE_NAME, - S3_PREFIX, CAMPAIGN_TABLE_NAME, + CHAMPION_TABLE_NAME, + COST_TABLE_NAME, IF_EXISTS_FAIL, IF_EXISTS_OVERWRITE, IF_EXISTS_SKIP, - SOL_TABLE_NAME, - TPM_TABLE_NAME, - COST_TABLE_NAME, QUALITY_TABLE_NAME, - CHAMPION_TABLE_NAME, ROOFLINE_TABLE_NAME, - S3Config, + S3_PREFIX, + SOL_TABLE_NAME, + TPM_TABLE_NAME, CampaignIncompleteError, + S3Config, build_atlas_table, build_campaign_row, - build_sol_table, - build_tpm_table, build_cost_table, build_quality_table, + build_sol_table, + build_tpm_table, parse_campaign_utc, parse_source_md, publish, @@ -57,7 +57,6 @@ from tools.perf_tune_report.perf_tune_report_cli import main from tools.perf_tune_report.schema import AtlasCell - pytest.importorskip("pyarrow") @@ -150,16 +149,12 @@ def _stage_campaign(tmp_path: Path, *, campaign_slug: str = "test-20260525T08165 def test_parse_campaign_utc_strips_slug_prefix(): - assert parse_campaign_utc("glm51-phase6-20260525T081650Z") == datetime( - 2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc - ) + assert parse_campaign_utc("glm51-phase6-20260525T081650Z") == datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC) def test_parse_campaign_utc_handles_utc_prefix(): # Canonical campaign_init layout is "-" (timestamp first). - assert parse_campaign_utc("20260529T175706Z-glm51-deepep-sweep") == datetime( - 2026, 5, 29, 17, 57, 6, tzinfo=timezone.utc - ) + assert parse_campaign_utc("20260529T175706Z-glm51-deepep-sweep") == datetime(2026, 5, 29, 17, 57, 6, tzinfo=UTC) def test_parse_campaign_utc_rejects_missing_suffix(): @@ -200,31 +195,71 @@ def test_build_atlas_table_schema_stable_across_extra_keys(): table = build_atlas_table(rows, campaign_id="campaign-20260525T081650Z") names = [f.name for f in table.schema] assert names == [ - "campaign_id", "cell_id", "model", "hardware", "quant", - "tensor_parallel", "parallel_strategy", "mtp", - "max_num_batched_tokens", "concurrency", "status", - "ttft_avg_ms", "request_throughput_avg", - "output_tps_per_user", "output_tps_per_gpu", + "campaign_id", + "cell_id", + "model", + "hardware", + "quant", + "tensor_parallel", + "parallel_strategy", + "mtp", + "max_num_batched_tokens", + "concurrency", + "status", + "ttft_avg_ms", + "request_throughput_avg", + "output_tps_per_user", + "output_tps_per_gpu", "total_tps_per_gpu", - "tpot_median_ms", "itl_avg_ms", - "mean_input_tokens", "mean_output_tokens", - "prefix_cache_hit_rate", "cache_mode", - "dataset", "cudagraph_mode", "gpu_memory_utilization", - "kv_cache_dtype", "image", "data_parallel", "pipeline_parallel", - "num_speculative_tokens", "async_scheduling", "max_num_seqs", - "enable_prefix_caching", "bench_backend", "variant_key", - "backend", "serving_engine", - "router_policy", "prefix_reuse", "per_replica_cache_hit", "acceptance_length", - "spec_accept_rate", - "kv_cache_tokens", "ep_mode", "dcgm_sm_active", "dcgm_dram_active", "dcgm_tensor_active", - "raw_path", "captured_at", "notes", "extra_json", - ] + "tpot_median_ms", + "itl_avg_ms", + "mean_input_tokens", + "mean_output_tokens", + "prefix_cache_hit_rate", + "cache_mode", + "dataset", + "cudagraph_mode", + "gpu_memory_utilization", + "kv_cache_dtype", + "image", + "data_parallel", + "pipeline_parallel", + "num_speculative_tokens", + "async_scheduling", + "max_num_seqs", + "enable_prefix_caching", + "bench_backend", + "variant_key", + "backend", + "serving_engine", + "router_policy", + "prefix_reuse", + "per_replica_cache_hit", + "acceptance_length", + "spec_accept_rate", + "kv_cache_tokens", + "ep_mode", + "dcgm_sm_active", + "dcgm_dram_active", + "dcgm_tensor_active", + "raw_path", + "captured_at", + "notes", + "extra_json", + ] assert table.num_rows == 3 # Serving-variant descriptor columns (2026-06-07) are present + variant_key populated. assert "variant_key" in names and "num_speculative_tokens" in names vt = build_atlas_table( - [_make_atlas_row(num_speculative_tokens=3, async_scheduling=True, - max_num_seqs=192, enable_prefix_caching=True, bench_backend="vllm")], + [ + _make_atlas_row( + num_speculative_tokens=3, + async_scheduling=True, + max_num_seqs=192, + enable_prefix_caching=True, + bench_backend="vllm", + ) + ], campaign_id="c-20260525T081650Z", ) assert vt.column("num_speculative_tokens").to_pylist() == [3] @@ -325,26 +360,33 @@ def test_build_atlas_table_true_grain_is_unique_after_numspec_lift(): _make_atlas_row(cell_id="cellA", concurrency=256, cache_mode="cold"), _make_atlas_row(cell_id="cellA", concurrency=256, cache_mode="warm"), # Same cell + c=1, MTP K2 vs K3 recorded only in extra. - _make_atlas_row(cell_id="cellB", concurrency=1, mtp=True, - num_speculative_tokens=None, extra={"num_speculative_tokens": 2}), - _make_atlas_row(cell_id="cellB", concurrency=1, mtp=True, - num_speculative_tokens=None, extra={"spec_decode_k": 3}), + _make_atlas_row( + cell_id="cellB", concurrency=1, mtp=True, num_speculative_tokens=None, extra={"num_speculative_tokens": 2} + ), + _make_atlas_row( + cell_id="cellB", concurrency=1, mtp=True, num_speculative_tokens=None, extra={"spec_decode_k": 3} + ), ] t = build_atlas_table(rows, campaign_id="c-20260525T081650Z") grain_cols = ( - "campaign_id", "cell_id", "concurrency", "mean_input_tokens", - "mean_output_tokens", "cache_mode", "num_speculative_tokens", + "campaign_id", + "cell_id", + "concurrency", + "mean_input_tokens", + "mean_output_tokens", + "cache_mode", + "num_speculative_tokens", ) cols = {n: t.column(n).to_pylist() for n in grain_cols} - keys = list(zip(*(cols[n] for n in grain_cols))) + keys = list(zip(*(cols[n] for n in grain_cols), strict=True)) assert len(keys) == len(set(keys)) == 6 # true grain: all distinct # ...whereas the legacy (campaign_id, cell_id, concurrency) key collapses them. - legacy = list(zip(cols["campaign_id"], cols["cell_id"], cols["concurrency"])) + legacy = list(zip(cols["campaign_id"], cols["cell_id"], cols["concurrency"], strict=True)) assert len(set(legacy)) == 3 < len(keys) def _quality_ts() -> datetime: - return datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc) + return datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC) def test_build_quality_table_long_format_flat_key_fallback(): @@ -358,16 +400,18 @@ def test_build_quality_table_long_format_flat_key_fallback(): "metric_kind": "train_accuracy_proxy", "acc_proxy_24k_samples": 0.311, "loss_24k_samples": 3.454, - "batch_size": 4, # hyperparameter -> excluded - "global_batch": 128, # hyperparameter -> excluded - "note": "eagle3", # non-numeric -> excluded + "batch_size": 4, # hyperparameter -> excluded + "global_batch": 128, # hyperparameter -> excluded + "note": "eagle3", # non-numeric -> excluded }, ), _make_atlas_row(cell_id="serving", extra={"max_num_seqs": 16}), # no metric_kind ] table = build_quality_table( - rows, campaign_id="x-20260525T081650Z", - captured_at_utc=_quality_ts(), published_at_utc=_quality_ts(), + rows, + campaign_id="x-20260525T081650Z", + captured_at_utc=_quality_ts(), + published_at_utc=_quality_ts(), ) assert set(table.column("metric_name").to_pylist()) == {"acc_proxy_24k_samples", "loss_24k_samples"} assert set(table.column("metric_kind").to_pylist()) == {"train_accuracy_proxy"} @@ -377,36 +421,64 @@ def test_build_quality_table_long_format_flat_key_fallback(): def test_build_quality_table_canonical_quality_metrics_subdict(): # Canonical convention: extra["quality_metrics"]={name: value} -- only those # numeric values are emitted (hyperparameters outside the sub-dict ignored). - rows = [_make_atlas_row(extra={ - "metric_kind": "acceptance", - "quality_metrics": {"acceptance_length": 3.2, "draft_hit_rate": 0.61}, - "global_batch": 128, - })] + rows = [ + _make_atlas_row( + extra={ + "metric_kind": "acceptance", + "quality_metrics": {"acceptance_length": 3.2, "draft_hit_rate": 0.61}, + "global_batch": 128, + } + ) + ] table = build_quality_table( - rows, campaign_id="x-20260525T081650Z", - captured_at_utc=_quality_ts(), published_at_utc=_quality_ts(), + rows, + campaign_id="x-20260525T081650Z", + captured_at_utc=_quality_ts(), + published_at_utc=_quality_ts(), ) assert set(table.column("metric_name").to_pylist()) == {"acceptance_length", "draft_hit_rate"} - assert table.column("metric_value").to_pylist() == [3.2, 0.61] or set(table.column("metric_value").to_pylist()) == {3.2, 0.61} + assert table.column("metric_value").to_pylist() == [3.2, 0.61] or set(table.column("metric_value").to_pylist()) == { + 3.2, + 0.61, + } def test_build_quality_table_empty_for_serving_only_campaign(): rows = [_make_atlas_row(extra={"max_num_seqs": 16})] table = build_quality_table( - rows, campaign_id="x-20260525T081650Z", - captured_at_utc=_quality_ts(), published_at_utc=_quality_ts(), + rows, + campaign_id="x-20260525T081650Z", + captured_at_utc=_quality_ts(), + published_at_utc=_quality_ts(), ) assert table.num_rows == 0 _SOL_SCHEMA_COLUMNS = [ - "campaign_id", "cell_id", "category", "sol_level", - "gpu_time_share_pct", "pct_sol", "bound", "ceiling_key", - "ceiling_value", "ceiling_units", "measured_value", "measured_units", - "attributed_bytes_total", "attributed_flops_total", - "arithmetic_intensity_flops_per_byte", "kernel_name", "hw_key", - "source_artifact", "source_artifact_sha256", "sol_ceilings_yaml_sha256", - "captured_at_utc", "published_at_utc", "focus", "sol_rigor", + "campaign_id", + "cell_id", + "category", + "sol_level", + "gpu_time_share_pct", + "pct_sol", + "bound", + "ceiling_key", + "ceiling_value", + "ceiling_units", + "measured_value", + "measured_units", + "attributed_bytes_total", + "attributed_flops_total", + "arithmetic_intensity_flops_per_byte", + "kernel_name", + "hw_key", + "source_artifact", + "source_artifact_sha256", + "sol_ceilings_yaml_sha256", + "captured_at_utc", + "published_at_utc", + "focus", + "sol_rigor", ] @@ -414,43 +486,66 @@ def _stage_sol_cells(campaign_dir: Path) -> None: """Add cells/cellA/{kernels.json,dcgm_correlation.json} SoL artifacts.""" cell = campaign_dir / "cells" / "cellA" cell.mkdir(parents=True, exist_ok=True) - (cell / "kernels.json").write_text(json.dumps({ - "schema_version": 1, - "captured_sources": ["zymtrace"], - "top_kernels": [], - "per_gpu": [], - "per_category": {"NCCL": 300, "MoE": 100, "FMHA": 100}, - "top_python_during_cuda": [], - })) - (cell / "dcgm_correlation.json").write_text(json.dumps({ - "schema_version": 1, - "captured_sources": ["dcgm"], - "hw_key": "b200_sm100", - "sweep_start_utc": "2026-05-25T06:18:00Z", - "sweep_end_utc": "2026-05-25T06:20:00Z", - "duration_s": 120.0, - "n_gpus": 8, - "dcgm_group_level": "prof", - "scrape_interval_s": 1.0, - "short_sweep_warning": False, - "resources": [ - {"peak_key": "hbm3e_tbps", "metric": "DCGM_FI_PROF_DRAM_ACTIVE", - "is_fallback": False, "n_gpus": 8, "measured_bytes_total": 5.0e13, - "measured_bytes_per_s": 4.32e11, "measured_tflops_avg": None, - "peak_per_gpu": 8.0, "peak_per_gpu_units": "TB/s", - "peak_aggregate": 64.0, "sol_pct": 5.4, "notes": []}, - ], - "queries": [], - "dry_run": False, - "per_category_attribution": [ - {"category": "NCCL", "time_share_pct": 60.0, - "attributed_bytes_total": 1.1e10, "attributed_flops_total": None, - "effective_bw_during_category_window": 1.35e8, - "effective_tflops_during_category_window": None, - "sol_pct_bw": 0.0075, "sol_pct_compute": None, - "bound": "bandwidth", "ceiling_metric": "nvlink5_tbps"}, - ], - })) + (cell / "kernels.json").write_text( + json.dumps( + { + "schema_version": 1, + "captured_sources": ["zymtrace"], + "top_kernels": [], + "per_gpu": [], + "per_category": {"NCCL": 300, "MoE": 100, "FMHA": 100}, + "top_python_during_cuda": [], + } + ) + ) + (cell / "dcgm_correlation.json").write_text( + json.dumps( + { + "schema_version": 1, + "captured_sources": ["dcgm"], + "hw_key": "b200_sm100", + "sweep_start_utc": "2026-05-25T06:18:00Z", + "sweep_end_utc": "2026-05-25T06:20:00Z", + "duration_s": 120.0, + "n_gpus": 8, + "dcgm_group_level": "prof", + "scrape_interval_s": 1.0, + "short_sweep_warning": False, + "resources": [ + { + "peak_key": "hbm3e_tbps", + "metric": "DCGM_FI_PROF_DRAM_ACTIVE", + "is_fallback": False, + "n_gpus": 8, + "measured_bytes_total": 5.0e13, + "measured_bytes_per_s": 4.32e11, + "measured_tflops_avg": None, + "peak_per_gpu": 8.0, + "peak_per_gpu_units": "TB/s", + "peak_aggregate": 64.0, + "sol_pct": 5.4, + "notes": [], + }, + ], + "queries": [], + "dry_run": False, + "per_category_attribution": [ + { + "category": "NCCL", + "time_share_pct": 60.0, + "attributed_bytes_total": 1.1e10, + "attributed_flops_total": None, + "effective_bw_during_category_window": 1.35e8, + "effective_tflops_during_category_window": None, + "sol_pct_bw": 0.0075, + "sol_pct_compute": None, + "bound": "bandwidth", + "ceiling_metric": "nvlink5_tbps", + }, + ], + } + ) + ) def test_build_sol_table_schema_stable_and_levels(tmp_path: Path): @@ -458,10 +553,13 @@ def test_build_sol_table_schema_stable_and_levels(tmp_path: Path): _stage_sol_cells(campaign_dir) rows = [_make_atlas_row()] table = build_sol_table( - campaign_dir, campaign_dir.name, rows, - captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc), - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), - focus="latency", sol_rigor="L3", + campaign_dir, + campaign_dir.name, + rows, + captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), + focus="latency", + sol_rigor="L3", ) assert [f.name for f in table.schema] == _SOL_SCHEMA_COLUMNS levels = set(table.column("sol_level").to_pylist()) @@ -480,10 +578,13 @@ def test_build_sol_table_schema_stable_and_levels(tmp_path: Path): def test_build_sol_table_empty_when_no_cells(tmp_path: Path): campaign_dir = _stage_campaign(tmp_path) # no cells/ dir table = build_sol_table( - campaign_dir, campaign_dir.name, [_make_atlas_row()], - captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc), - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), - focus="mixed", sol_rigor="none", + campaign_dir, + campaign_dir.name, + [_make_atlas_row()], + captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), + focus="mixed", + sol_rigor="none", ) assert table.num_rows == 0 assert [f.name for f in table.schema] == _SOL_SCHEMA_COLUMNS @@ -495,9 +596,10 @@ def test_build_tpm_table_peak_only_without_sla(): _make_atlas_row(concurrency=2, output_tps_per_gpu=18.81), ] table = build_tpm_table( - rows, "x-20260525T081650Z", - captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc), - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), + rows, + "x-20260525T081650Z", + captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), ) ops = set(table.column("operating_point").to_pylist()) assert ops == {"peak"} # no SLA thresholds -> peak-only @@ -510,9 +612,10 @@ def test_build_tpm_table_emits_sla_rows_with_thresholds(): ] # ttft<=200 -> only the c=1 row qualifies; peak is the c=2 row. table = build_tpm_table( - rows, "x-20260525T081650Z", - captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc), - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), + rows, + "x-20260525T081650Z", + captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), ttft_sla_ms=200.0, ) d = table.to_pylist() @@ -529,9 +632,11 @@ def test_build_cost_table_usd_per_1m_tokens(tmp_path: Path): # $/1M out = 4.5*1e6/(10*3600) = 125.0 rows = [_make_atlas_row(output_tps_per_gpu=10.0, total_tps_per_gpu=40.0)] table = build_cost_table( - campaign_dir, rows, campaign_dir.name, - captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc), - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), + campaign_dir, + rows, + campaign_dir.name, + captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), usd_per_gpu_hour={"B200": 4.50}, ) d = table.to_pylist() @@ -547,9 +652,11 @@ def test_build_cost_table_usd_per_1m_tokens(tmp_path: Path): def test_build_cost_table_cost_null_without_config(tmp_path: Path): campaign_dir = _stage_campaign(tmp_path) table = build_cost_table( - campaign_dir, [_make_atlas_row(output_tps_per_gpu=10.0)], campaign_dir.name, - captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc), - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), + campaign_dir, + [_make_atlas_row(output_tps_per_gpu=10.0)], + campaign_dir.name, + captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), ) d = table.to_pylist() assert d[0]["usd_per_1m_output_tokens"] is None @@ -561,16 +668,26 @@ def test_build_cost_table_tokens_per_watt_from_dcgm_power(tmp_path: Path): # Stage a per-cell dcgm_correlation.json carrying power_watts_per_gpu. cell = campaign_dir / "cells" / "cellA" cell.mkdir(parents=True, exist_ok=True) - (cell / "dcgm_correlation.json").write_text(json.dumps({ - "schema_version": 1, "captured_sources": ["dcgm"], "hw_key": "b200_sm100", - "resources": [], "queries": [], "power_watts_per_gpu": 700.0, - })) + (cell / "dcgm_correlation.json").write_text( + json.dumps( + { + "schema_version": 1, + "captured_sources": ["dcgm"], + "hw_key": "b200_sm100", + "resources": [], + "queries": [], + "power_watts_per_gpu": 700.0, + } + ) + ) # The peak point's row must originate from cellA so the power join hits. rows = [_make_atlas_row(cell_id="cellA", output_tps_per_gpu=350.0)] table = build_cost_table( - campaign_dir, rows, campaign_dir.name, - captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc), - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), + campaign_dir, + rows, + campaign_dir.name, + captured_at_utc=datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), ) d = table.to_pylist() assert d[0]["power_watts_per_gpu"] == 700.0 @@ -587,8 +704,7 @@ def test_publish_reads_tpm_config_block(tmp_path: Path): # config block sets both thresholds the rows meet. rows = [ _make_atlas_row(concurrency=1, ttft_avg_ms=182.0, tpot_median_ms=20.0), - _make_atlas_row(concurrency=2, ttft_avg_ms=264.0, tpot_median_ms=22.0, - output_tps_per_gpu=18.81), + _make_atlas_row(concurrency=2, ttft_avg_ms=264.0, tpot_median_ms=22.0, output_tps_per_gpu=18.81), ] with (campaign_dir / "atlas.jsonl").open("w", encoding="utf-8") as f: for row in rows: @@ -639,7 +755,7 @@ def test_build_campaign_row_reads_sourcemd_provenance(tmp_path: Path): table = build_campaign_row( campaign_dir, rows, - published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=timezone.utc), + published_at_utc=datetime(2026, 5, 25, 16, 0, 0, tzinfo=UTC), publisher_operator="ci-bot", publisher_host="ci-runner-1", ) @@ -653,7 +769,7 @@ def test_build_campaign_row_reads_sourcemd_provenance(tmp_path: Path): assert row["atlas_row_count"] == 2 assert row["publisher_operator"] == "ci-bot" assert row["publisher_host"] == "ci-runner-1" - assert row["captured_at_utc"] == datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc) + assert row["captured_at_utc"] == datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC) # config_yaml + SOURCE.md were created by _stage_campaign, so digests are non-empty. assert len(row["config_yaml_sha256"]) == 64 assert len(row["source_md_sha256"]) == 64 @@ -665,12 +781,9 @@ def test_build_campaign_row_reads_sourcemd_provenance(tmp_path: Path): def test_s3_key_uses_hive_layout(): - captured = datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc) + captured = datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC) atlas_key = s3_key_for(ATLAS_TABLE_NAME, "glm51-phase6-20260525T081650Z", captured) - assert atlas_key == ( - f"{S3_PREFIX}/atlas_v1/dt=2026-05-25/" - "campaign=glm51-phase6-20260525T081650Z/part-0.parquet" - ) + assert atlas_key == (f"{S3_PREFIX}/atlas_v1/dt=2026-05-25/campaign=glm51-phase6-20260525T081650Z/part-0.parquet") campaign_key = s3_key_for(CAMPAIGN_TABLE_NAME, "glm51-phase6-20260525T081650Z", captured) assert campaign_key.endswith("campaign_v1/dt=2026-05-25/campaign=glm51-phase6-20260525T081650Z/part-0.parquet") @@ -722,9 +835,7 @@ def put_object( Body: Any, IfNoneMatch: str | None = None, ) -> dict[str, Any]: - self.put_attempts.append( - {"bucket": Bucket, "key": Key, "if_none_match": IfNoneMatch} - ) + self.put_attempts.append({"bucket": Bucket, "key": Key, "if_none_match": IfNoneMatch}) if self.put_error is not None: raise self.put_error self.put_calls.append({"bucket": Bucket, "key": Key}) @@ -925,7 +1036,7 @@ def test_publish_real_run_uploads_eight_objects(tmp_path: Path): def test_publish_skip_on_existing_keys(tmp_path: Path): campaign_dir = _stage_campaign(tmp_path) - captured = datetime(2026, 5, 25, 8, 16, 50, tzinfo=timezone.utc) + captured = datetime(2026, 5, 25, 8, 16, 50, tzinfo=UTC) pre_existing = { s3_key_for(ATLAS_TABLE_NAME, campaign_dir.name, captured), s3_key_for(CAMPAIGN_TABLE_NAME, campaign_dir.name, captured), @@ -1035,7 +1146,9 @@ def test_publish_strict_refuses_zero_plot_ready(tmp_path: Path): campaign_dir = _stage_campaign(tmp_path) _write_status(campaign_dir, plot_ready_points=0) # focus defaults "mixed" with pytest.raises(CampaignIncompleteError, match="throughput-scatter"): - publish(campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, s3_client_factory=lambda _cfg: _FakeS3Client()) + publish( + campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, s3_client_factory=lambda _cfg: _FakeS3Client() + ) def test_campaign_row_records_focus_and_sol_rigor(tmp_path: Path): @@ -1050,8 +1163,7 @@ def test_campaign_row_records_focus_and_sol_rigor(tmp_path: Path): def test_campaign_row_has_completeness_columns(tmp_path: Path): campaign_dir = _stage_campaign(tmp_path) table = build_campaign_row(campaign_dir, _read_atlas_rows(campaign_dir)) - for col in ("sol_complete", "plot_ready_points", "omitted_pages", "partial_pages", - "sol_per_arm_complete"): + for col in ("sol_complete", "plot_ready_points", "omitted_pages", "partial_pages", "sol_per_arm_complete"): assert col in table.column_names assert table.column("sol_complete")[0].as_py() is True assert table.column("plot_ready_points")[0].as_py() == 3 @@ -1085,10 +1197,7 @@ def test_campaign_row_records_partial_pages(tmp_path: Path): ], ) table = build_campaign_row(campaign_dir, _read_atlas_rows(campaign_dir)) - assert ( - table.column("partial_pages")[0].as_py() - == "Byte-grounded per-kernel SoL scatter (page 5)" - ) + assert table.column("partial_pages")[0].as_py() == "Byte-grounded per-kernel SoL scatter (page 5)" def test_campaign_row_defaults_experiment_id_to_campaign_id(tmp_path: Path): @@ -1162,12 +1271,12 @@ def test_publish_strict_blocks_verdict_without_pinned_source(tmp_path: Path): campaign_dir = _stage_campaign(tmp_path) _write_status(campaign_dir, focus="latency") (campaign_dir / "verdict.json").write_text( - json.dumps({"tier": "verdict", "trials": 3, "same_node": True, - "baseline_named": True}) + json.dumps({"tier": "verdict", "trials": 3, "same_node": True, "baseline_named": True}) ) with pytest.raises(CampaignIncompleteError, match="provenance block"): - publish(campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, - s3_client_factory=lambda _cfg: _FakeS3Client()) + publish( + campaign_dir, cfg=_stub_cfg(), dry_run=True, strict=True, s3_client_factory=lambda _cfg: _FakeS3Client() + ) def test_source_problems_blocks_unpinned_verdict(tmp_path: Path): @@ -1258,8 +1367,10 @@ def test_resolve_s3_config_missing_keys_exits(monkeypatch): monkeypatch.delenv("PERFLAKE_LAKE_S3_SECRET_KEY", raising=False) with pytest.raises(SystemExit, match="S3 access/secret key missing"): resolve_s3_config( - endpoint=None, bucket=None, - access_key_file=None, secret_key_file=None, + endpoint=None, + bucket=None, + access_key_file=None, + secret_key_file=None, ) @@ -1276,13 +1387,16 @@ def test_cli_dry_run_succeeds_without_s3(monkeypatch, tmp_path: Path, capsys): # --no-strict: this fixture is intentionally SoL-incomplete (no cells/*/ SoL # artifacts -> sol_v1 empty); publish defaults strict now, so opt out to # exercise the first-class intentional-gap publish path. - rc = main([ - "publish_to_lake", - "--campaign", campaign_dir.name, - "--dry-run", - "--no-strict", - "--json", - ]) + rc = main( + [ + "publish_to_lake", + "--campaign", + campaign_dir.name, + "--dry-run", + "--no-strict", + "--json", + ] + ) assert rc == 0 out = capsys.readouterr().out envelope = json.loads(out) @@ -1320,10 +1434,7 @@ def test_publish_appends_lake_provenance_to_evidence_bundle(monkeypatch, tmp_pat # Campaign that points back at the bundle. campaign_dir = _stage_campaign(tmp_path) (campaign_dir / "SOURCE.md").write_text( - "# Campaign\n\n" - "- captured_at: 20260525T081650Z\n" - "- config: /tmp/test.yaml\n" - f"- evidence_bundle_path: {bundle}\n" + f"# Campaign\n\n- captured_at: 20260525T081650Z\n- config: /tmp/test.yaml\n- evidence_bundle_path: {bundle}\n" ) monkeypatch.setenv("PERFREPORT_CAMPAIGNS_DIR", str(tmp_path)) monkeypatch.setenv("PERFLAKE_LAKE_S3_ACCESS_KEY", "envAK") @@ -1333,30 +1444,36 @@ def test_publish_appends_lake_provenance_to_evidence_bundle(monkeypatch, tmp_pat monkeypatch.setattr(lw, "_make_s3_client", lambda cfg: _FakeS3Client()) # --no-strict: SoL-incomplete fixture (intentional-gap publish path). - rc = main([ - "publish_to_lake", - "--campaign", campaign_dir.name, - "--no-strict", - "--i-understand-this-publishes-externally", - "--json", - ]) + rc = main( + [ + "publish_to_lake", + "--campaign", + campaign_dir.name, + "--no-strict", + "--i-understand-this-publishes-externally", + "--json", + ] + ) assert rc == 0 updated = (bundle / "SOURCE.md").read_text() assert f"campaign={campaign_dir.name}" in updated assert "atlas_v1: s3://perf-lake/" in updated assert "campaign_v1: s3://perf-lake/" in updated # Idempotent: a second publish does not duplicate the block. - rc2 = main([ - "publish_to_lake", - "--campaign", campaign_dir.name, - "--if-exists", "overwrite", - "--no-strict", - "--i-understand-this-publishes-externally", - "--json", - ]) + rc2 = main( + [ + "publish_to_lake", + "--campaign", + campaign_dir.name, + "--if-exists", + "overwrite", + "--no-strict", + "--i-understand-this-publishes-externally", + "--json", + ] + ) assert rc2 == 0 - assert (bundle / "SOURCE.md").read_text().count( - "## Perf-lake publish (auto-appended") == 1 + assert (bundle / "SOURCE.md").read_text().count("## Perf-lake publish (auto-appended") == 1 def test_cli_strict_is_default_refuses_incomplete(monkeypatch, tmp_path: Path, capsys): @@ -1371,6 +1488,15 @@ def test_cli_strict_is_default_refuses_incomplete(monkeypatch, tmp_path: Path, c assert rc == 2 +def test_krhpa_readers_ignore_non_mapping_yaml(tmp_path: Path): + import tools.perf_tune_report.lake_writer as lake_writer + + (tmp_path / "config.yaml").write_text("- not\n- a mapping\n") + + assert lake_writer._read_krhpa(tmp_path) is None + assert lake_writer._read_krhpa_exempt_reason(tmp_path) == "" + + def test_cli_real_publish_requires_current_command_ack(monkeypatch, tmp_path: Path, capsys): campaign_dir = _stage_campaign(tmp_path) monkeypatch.setenv("PERFREPORT_CAMPAIGNS_DIR", str(tmp_path)) @@ -1385,12 +1511,15 @@ def test_cli_missing_atlas_fails_with_clear_message(monkeypatch, tmp_path: Path, monkeypatch.setenv("PERFREPORT_CAMPAIGNS_DIR", str(tmp_path)) monkeypatch.setenv("PERFLAKE_LAKE_S3_ACCESS_KEY", "envAK") monkeypatch.setenv("PERFLAKE_LAKE_S3_SECRET_KEY", "envSK") - rc = main([ - "publish_to_lake", - "--campaign", campaign_dir.name, - "--dry-run", - "--json", - ]) + rc = main( + [ + "publish_to_lake", + "--campaign", + campaign_dir.name, + "--dry-run", + "--json", + ] + ) assert rc == 2 err = capsys.readouterr().err assert "atlas.jsonl not found" in err diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_raw_bench_compare.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_raw_bench_compare.py index f76900c..d76afc6 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_raw_bench_compare.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_raw_bench_compare.py @@ -7,7 +7,6 @@ from __future__ import annotations -import json import sys from pathlib import Path @@ -19,14 +18,12 @@ sys.path.insert(0, str(REPO_ROOT)) from tools.perf_tune_report.raw_bench_compare import ( - BundleSpec, RawBenchCompareManifestMalformed, _parse_sweep_file, load_manifest, render_comparison, ) - _SWEEP_TEXT_TEMPLATE = """ ============ Serving Benchmark Result ============ Successful requests: {successful} @@ -49,25 +46,29 @@ """ -def _write_sweep(bundle_dir: Path, c: int, *, output_tps: float, tpot_median: float, ttft_median: float = 200.0) -> Path: +def _write_sweep( + bundle_dir: Path, c: int, *, output_tps: float, tpot_median: float, ttft_median: float = 200.0 +) -> Path: """Write a synthetic sweep-c.txt with the vllm bench serve format.""" raw = bundle_dir / "raw" raw.mkdir(parents=True, exist_ok=True) path = raw / f"sweep-c{c}.txt" - path.write_text(_SWEEP_TEXT_TEMPLATE.format( - c=c, - successful=c * 32, - duration_s=300.0, - req_per_s=output_tps / 100.0, - output_tps=output_tps, - total_tps=output_tps * 1.5, - ttft_mean=ttft_median * 1.2, - ttft_median=ttft_median, - ttft_p99=ttft_median * 1.8, - tpot_mean=tpot_median * 1.05, - tpot_median=tpot_median, - tpot_p99=tpot_median * 1.3, - )) + path.write_text( + _SWEEP_TEXT_TEMPLATE.format( + c=c, + successful=c * 32, + duration_s=300.0, + req_per_s=output_tps / 100.0, + output_tps=output_tps, + total_tps=output_tps * 1.5, + ttft_mean=ttft_median * 1.2, + ttft_median=ttft_median, + ttft_p99=ttft_median * 1.8, + tpot_mean=tpot_median * 1.05, + tpot_median=tpot_median, + tpot_p99=tpot_median * 1.3, + ) + ) return path @@ -192,10 +193,14 @@ def test_load_manifest_raises_on_missing_bundles_key(tmp_path: Path): def test_load_manifest_raises_on_missing_required_bundle_field(tmp_path: Path): p = tmp_path / "incomplete.yaml" - p.write_text(_yaml.dump({ - "schema_version": 1, - "bundles": [{"glob": "x", "label": "y"}], # missing 'short' - })) + p.write_text( + _yaml.dump( + { + "schema_version": 1, + "bundles": [{"glob": "x", "label": "y"}], # missing 'short' + } + ) + ) with pytest.raises(RawBenchCompareManifestMalformed, match="missing 'short'"): load_manifest(p) @@ -205,14 +210,18 @@ def test_load_manifest_resolves_bundles_root_relative_to_manifest(tmp_path: Path # Write manifest at sibling location with bundles_root as relative path. relative_root = bundles_root.relative_to(tmp_path) p = tmp_path / "relative_manifest.yaml" - p.write_text(_yaml.dump({ - "schema_version": 1, - "bundles_root": str(relative_root), # relative - "bundles": [ - {"glob": "model-baseline-*", "label": "B", "short": "b"}, - {"glob": "model-champion-*", "label": "C", "short": "c"}, - ], - })) + p.write_text( + _yaml.dump( + { + "schema_version": 1, + "bundles_root": str(relative_root), # relative + "bundles": [ + {"glob": "model-baseline-*", "label": "B", "short": "b"}, + {"glob": "model-champion-*", "label": "C", "short": "c"}, + ], + } + ) + ) _, specs = load_manifest(p) assert all(s.bundle_path is not None for s in specs) @@ -270,10 +279,14 @@ def test_render_comparison_raises_when_no_bundles_have_data(tmp_path: Path): empty_root = tmp_path / "empty" empty_root.mkdir() p = tmp_path / "empty_manifest.yaml" - p.write_text(_yaml.dump({ - "schema_version": 1, - "bundles_root": str(empty_root), - "bundles": [{"glob": "missing-*", "label": "x", "short": "x"}], - })) - with pytest.raises(ValueError, match="no bundles .* yielded parseable rows"): + p.write_text( + _yaml.dump( + { + "schema_version": 1, + "bundles_root": str(empty_root), + "bundles": [{"glob": "missing-*", "label": "x", "short": "x"}], + } + ) + ) + with pytest.raises(ValueError, match=r"no bundles .* yielded parseable rows"): render_comparison(p, tmp_path / "out.pdf") diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_math.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_math.py index 1b82b88..fc84e5f 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_math.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_math.py @@ -23,8 +23,8 @@ def test_glm51_registry_matches_exemplar(): ai_c1 = s.decode_arithmetic_intensity(1, 512, "NVFP4", "fp8") ai_c192 = s.decode_arithmetic_intensity(192, 512, "NVFP4", "fp8") - assert 3.0 < ai_c1 < 4.0 # exemplar 3.4 - assert 30.0 < ai_c192 < 45.0 # exemplar ~40 + assert 3.0 < ai_c1 < 4.0 # exemplar 3.4 + assert 30.0 < ai_c192 < 45.0 # exemplar ~40 # decode is memory-bound across the whole feasible serving range for c in (1, 2, 4, 8, 16, 32, 64, 128, 192): assert s.decode_arithmetic_intensity(c, 512, "NVFP4", "fp8") < RIDGE_GB300_NVFP4 @@ -32,8 +32,8 @@ def test_glm51_registry_matches_exemplar(): # prefill climbs toward the ridge as the chunk grows (compute regime) p512 = s.prefill_arithmetic_intensity(512, "NVFP4") p8192 = s.prefill_arithmetic_intensity(8192, "NVFP4") - assert p8192 > p512 > ai_c192 # prefill AI >> decode AI - assert p8192 > 1000 # near/above the ridge + assert p8192 > p512 > ai_c192 # prefill AI >> decode AI + assert p8192 > 1000 # near/above the ridge # FLOP/token is positive and the active-weight bytes amortize with the union assert s.flop_per_token > 0 @@ -49,8 +49,12 @@ def test_decode_ai_monotonic_in_concurrency(): def test_from_hf_config_dense_gqa(): """A Llama-3.1-8B-style dense GQA config -> dense shape, KV = 2*kv_heads*head_dim.""" cfg = { - "hidden_size": 4096, "num_hidden_layers": 32, "num_attention_heads": 32, - "num_key_value_heads": 8, "vocab_size": 128256, "intermediate_size": 14336, + "hidden_size": 4096, + "num_hidden_layers": 32, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "vocab_size": 128256, + "intermediate_size": 14336, } s = rm.from_hf_config(cfg, name="meta-llama/Llama-3.1-8B") assert not s.is_moe and not s.is_mla @@ -64,12 +68,21 @@ def test_from_hf_config_dense_gqa(): def test_from_hf_config_detects_moe_and_mla(): cfg = { - "hidden_size": 7168, "num_hidden_layers": 61, "num_attention_heads": 128, - "vocab_size": 129280, "intermediate_size": 18432, - "n_routed_experts": 256, "num_experts_per_tok": 8, "moe_intermediate_size": 2048, - "n_shared_experts": 1, "first_k_dense_replace": 3, - "kv_lora_rank": 512, "q_lora_rank": 1536, "qk_nope_head_dim": 128, - "qk_rope_head_dim": 64, "v_head_dim": 128, + "hidden_size": 7168, + "num_hidden_layers": 61, + "num_attention_heads": 128, + "vocab_size": 129280, + "intermediate_size": 18432, + "n_routed_experts": 256, + "num_experts_per_tok": 8, + "moe_intermediate_size": 2048, + "n_shared_experts": 1, + "first_k_dense_replace": 3, + "kv_lora_rank": 512, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "v_head_dim": 128, } s = rm.from_hf_config(cfg, name="deepseek-ai/DeepSeek-V3") assert s.is_moe and s.is_mla @@ -84,4 +97,6 @@ def test_summary_roundtrip(): assert summ["is_moe"] and summ["flop_per_token"] > 0 d = rm.shape_to_dict(s) s2 = rm.shape_from_dict(d) - assert s2.decode_arithmetic_intensity(8, 512, "NVFP4", "fp8") == s.decode_arithmetic_intensity(8, 512, "NVFP4", "fp8") + assert s2.decode_arithmetic_intensity(8, 512, "NVFP4", "fp8") == s.decode_arithmetic_intensity( + 8, 512, "NVFP4", "fp8" + ) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_sweep.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_sweep.py index 57e283e..03830e4 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_sweep.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_roofline_sweep.py @@ -1,7 +1,7 @@ """Tests for the always-on prefill/decode roofline sweep importer + page.""" import json -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from tools.perf_tune_report.importers.roofline_sweep import import_roofline_sweep_bundle @@ -9,16 +9,30 @@ def _cell(c, isl, osl, tpot, ttft, out_tps, sm, ten, dram): return { - "tag": f"c{c}", "c": c, "isl": isl, "osl": osl, "num_prompts": max(c, 6), + "tag": f"c{c}", + "c": c, + "isl": isl, + "osl": osl, + "num_prompts": max(c, 6), "bench": { - "duration": 20.0, "completed": c, "total_input_tokens": isl * c, - "total_output_tokens": osl * c, "request_throughput": out_tps / max(osl, 1), - "output_throughput": out_tps, "total_token_throughput": out_tps * 1.5, - "median_ttft_ms": ttft, "median_tpot_ms": tpot, "median_itl_ms": tpot, + "duration": 20.0, + "completed": c, + "total_input_tokens": isl * c, + "total_output_tokens": osl * c, + "request_throughput": out_tps / max(osl, 1), + "output_throughput": out_tps, + "total_token_throughput": out_tps * 1.5, + "median_ttft_ms": ttft, + "median_tpot_ms": tpot, + "median_itl_ms": tpot, }, "dcgm_steady": { - "sm_active_mean": sm, "tensor_active_mean": ten, "dram_active_mean": dram, - "fp16_active_mean": 0.0, "nvlink_tx_Bps_mean": 1e9, "nvlink_rx_Bps_mean": 1e9, + "sm_active_mean": sm, + "tensor_active_mean": ten, + "dram_active_mean": dram, + "fp16_active_mean": 0.0, + "nvlink_tx_Bps_mean": 1e9, + "nvlink_rx_Bps_mean": 1e9, "dmon_samples": 100, }, } @@ -45,9 +59,15 @@ def test_import_roofline_sweep_writes_cells_and_dcgm_util(tmp_path): camp = tmp_path / "camp" camp.mkdir() r = import_roofline_sweep_bundle( - bundle, camp, overrides={"tensor_parallel": 4, "hardware": "GB300", - "quant": "NVFP4", "model": "zai-org/GLM-5.1", - "cell_id": "glm51-tp4"}, + bundle, + camp, + overrides={ + "tensor_parallel": 4, + "hardware": "GB300", + "quant": "NVFP4", + "model": "zai-org/GLM-5.1", + "cell_id": "glm51-tp4", + }, ) assert r.decode_points == 2 and r.prefill_points == 2 dec = camp / "cells" / "glm51-tp4-decode" / "normalized.json" @@ -83,9 +103,15 @@ def test_import_roofline_sweep_unknown_model_degrades_cleanly(tmp_path): camp = tmp_path / "camp" camp.mkdir() r = import_roofline_sweep_bundle( - bundle, camp, overrides={"tensor_parallel": 4, "hardware": "GB300", - "quant": "NVFP4", "model": "acme/Unknown-42B", - "cell_id": "unk"}, + bundle, + camp, + overrides={ + "tensor_parallel": 4, + "hardware": "GB300", + "quant": "NVFP4", + "model": "acme/Unknown-42B", + "cell_id": "unk", + }, ) assert r.decode_points == 2 art = json.loads((camp / "cells" / "unk-decode" / "roofline_sweep.json").read_text()) @@ -98,7 +124,8 @@ def test_import_roofline_sweep_full_context_descriptor_overrides(tmp_path): camp = tmp_path / "camp" camp.mkdir() import_roofline_sweep_bundle( - bundle, camp, + bundle, + camp, overrides={ "tensor_parallel": 4, "hardware": "GB300", @@ -130,9 +157,8 @@ def test_import_roofline_sweep_full_context_descriptor_overrides(tmp_path): def test_build_roofline_v1_lake_table(tmp_path, monkeypatch): """roofline_v1 flattens every (c, ISL) point with analytical coords + ceilings so Superset can render the roofline scatter + util-vs-C lines from the lake.""" - from tools.perf_tune_report import lake_writer + from tools.perf_tune_report import aggregator, lake_writer from tools.perf_tune_report.schema import read_jsonl - from tools.perf_tune_report import aggregator yaml_path = tmp_path / "perf-tune-report" / "configs" / "sol-ceilings.yaml" yaml_path.parent.mkdir(parents=True) @@ -148,15 +174,20 @@ def test_build_roofline_v1_lake_table(tmp_path, monkeypatch): camp = tmp_path / "camp" camp.mkdir() import_roofline_sweep_bundle( - bundle, camp, overrides={"tensor_parallel": 4, "hardware": "GB300", - "quant": "NVFP4", "model": "zai-org/GLM-5.1", - "cell_id": "glm51-tp4"}, + bundle, + camp, + overrides={ + "tensor_parallel": 4, + "hardware": "GB300", + "quant": "NVFP4", + "model": "zai-org/GLM-5.1", + "cell_id": "glm51-tp4", + }, ) aggregator.aggregate(camp) rows = read_jsonl(camp / "atlas.jsonl") - now = datetime.now(timezone.utc) - table = lake_writer.build_roofline_table(camp, "camp", rows, - captured_at_utc=now, published_at_utc=now) + now = datetime.now(UTC) + table = lake_writer.build_roofline_table(camp, "camp", rows, captured_at_utc=now, published_at_utc=now) # 2 decode + 2 prefill points assert table.num_rows == 4 d = table.to_pylist() @@ -176,19 +207,27 @@ def test_build_roofline_v1_lake_table(tmp_path, monkeypatch): def test_source_links_from_provenance_and_registry(): from tools.perf_tune_report import provenance + prov = { "schema": "experiment_provenance_v1", "identity": {"run_id": "x", "title": "GLM-5.1 nvfp4-KV roofline"}, "source": [ - {"repo": "example/vllm", "branch": "feature/nvfp4-kv", - "commit": "6554db7dc", "delivery": "overlay", - "image": "infr/vllm:v2.12.3"}, + { + "repo": "example/vllm", + "branch": "feature/nvfp4-kv", + "commit": "6554db7dc", + "delivery": "overlay", + "image": "infr/vllm:v2.12.3", + }, {"repo": "example/perf-tune-glm51", "commit": "eafb4b4"}, ], } - registry = {"repo": "example/vllm", "branches": [ - {"branch": "feature/nvfp4-kv", "purpose": "NVFP4 KV + sparse-MLA decode"}, - ]} + registry = { + "repo": "example/vllm", + "branches": [ + {"branch": "feature/nvfp4-kv", "purpose": "NVFP4 KV + sparse-MLA decode"}, + ], + } links = provenance.source_links(prov, registry) assert len(links) == 2 assert links[0]["url"] == "https://github.com/example/vllm/commit/6554db7dc" @@ -201,6 +240,7 @@ def test_source_links_from_provenance_and_registry(): def test_source_page_renders_when_provenance_present(tmp_path, monkeypatch): import matplotlib + matplotlib.use("Agg") from tools.perf_tune_report import aggregator from tools.perf_tune_report.renderer import render_report @@ -217,17 +257,28 @@ def test_source_page_renders_when_provenance_present(tmp_path, monkeypatch): camp = tmp_path / "camp" camp.mkdir() import_roofline_sweep_bundle( - bundle, camp, overrides={"tensor_parallel": 4, "hardware": "GB300", - "quant": "NVFP4", "model": "zai-org/GLM-5.1", - "cell_id": "glm51-tp4"}, + bundle, + camp, + overrides={ + "tensor_parallel": 4, + "hardware": "GB300", + "quant": "NVFP4", + "model": "zai-org/GLM-5.1", + "cell_id": "glm51-tp4", + }, ) # campaign_init normally writes provenance.json; stage it directly here - (camp / "provenance.json").write_text(json.dumps({ - "schema": "experiment_provenance_v1", - "identity": {"run_id": "glm51-roofline", "title": "GLM-5.1 roofline"}, - "source": [{"repo": "example/vllm", "branch": "feature/nvfp4-kv", - "commit": "6554db7dc", "delivery": "overlay"}], - })) + (camp / "provenance.json").write_text( + json.dumps( + { + "schema": "experiment_provenance_v1", + "identity": {"run_id": "glm51-roofline", "title": "GLM-5.1 roofline"}, + "source": [ + {"repo": "example/vllm", "branch": "feature/nvfp4-kv", "commit": "6554db7dc", "delivery": "overlay"} + ], + } + ) + ) aggregator.aggregate(camp) status = render_report.render_report(camp / "atlas.jsonl", camp / "report.pdf", title="t") assert "source under test" in status.to_dict()["rendered_pages"] @@ -240,29 +291,35 @@ def test_roofline_mandatory_gate(): omitted = "Prefill/Decode roofline (page 7)" # throughput campaign with plot-ready points but page 7 omitted -> gated - rs = RenderStatusSummary(rendered=True, sol_complete=True, plot_ready_points=9, - omitted_pages=omitted, focus="throughput") + rs = RenderStatusSummary( + rendered=True, sol_complete=True, plot_ready_points=9, omitted_pages=omitted, focus="throughput" + ) assert roofline_problems(rs) # mixed focus too - rs_mixed = RenderStatusSummary(rendered=True, sol_complete=True, plot_ready_points=5, - omitted_pages=omitted, focus="mixed") + rs_mixed = RenderStatusSummary( + rendered=True, sol_complete=True, plot_ready_points=5, omitted_pages=omitted, focus="mixed" + ) assert roofline_problems(rs_mixed) # latency-only run -> exempt - rs_lat = RenderStatusSummary(rendered=True, sol_complete=True, plot_ready_points=0, - omitted_pages=omitted, focus="latency") + rs_lat = RenderStatusSummary( + rendered=True, sol_complete=True, plot_ready_points=0, omitted_pages=omitted, focus="latency" + ) assert roofline_problems(rs_lat) == [] # throughput but 0 plot-ready points (e.g. all cells failed) -> exempt - rs_zero = RenderStatusSummary(rendered=True, sol_complete=False, plot_ready_points=0, - omitted_pages=omitted, focus="throughput") + rs_zero = RenderStatusSummary( + rendered=True, sol_complete=False, plot_ready_points=0, omitted_pages=omitted, focus="throughput" + ) assert roofline_problems(rs_zero) == [] # page 7 present (not omitted) -> no problem - rs_ok = RenderStatusSummary(rendered=True, sol_complete=True, plot_ready_points=9, - omitted_pages="", focus="throughput") + rs_ok = RenderStatusSummary( + rendered=True, sol_complete=True, plot_ready_points=9, omitted_pages="", focus="throughput" + ) assert roofline_problems(rs_ok) == [] def test_roofline_page_renders_and_sets_sol_l3(tmp_path, monkeypatch): import matplotlib + matplotlib.use("Agg") from tools.perf_tune_report import aggregator from tools.perf_tune_report.renderer import render_report @@ -285,13 +342,18 @@ def test_roofline_page_renders_and_sets_sol_l3(tmp_path, monkeypatch): camp = tmp_path / "camp" camp.mkdir() import_roofline_sweep_bundle( - bundle, camp, overrides={"tensor_parallel": 4, "hardware": "GB300", - "quant": "NVFP4", "model": "zai-org/GLM-5.1", - "cell_id": "glm51-tp4"}, + bundle, + camp, + overrides={ + "tensor_parallel": 4, + "hardware": "GB300", + "quant": "NVFP4", + "model": "zai-org/GLM-5.1", + "cell_id": "glm51-tp4", + }, ) aggregator.aggregate(camp) - status = render_report.render_report(camp / "atlas.jsonl", camp / "report.pdf", - title="roofline test") + status = render_report.render_report(camp / "atlas.jsonl", camp / "report.pdf", title="roofline test") d = status.to_dict() assert "prefill/decode roofline (page 7)" in d["rendered_pages"] assert d["sol_complete"] is True @@ -302,6 +364,7 @@ def test_roofline_page_renders_and_sets_sol_l3(tmp_path, monkeypatch): def test_per_arm_coverage_uses_extra_arm_artifact_dir(tmp_path, monkeypatch): import matplotlib + matplotlib.use("Agg") from tools.perf_tune_report import aggregator from tools.perf_tune_report.renderer import render_report @@ -321,16 +384,31 @@ def test_per_arm_coverage_uses_extra_arm_artifact_dir(tmp_path, monkeypatch): camp = tmp_path / "camp" camp.mkdir() import_roofline_sweep_bundle( - bundle, camp, overrides={"tensor_parallel": 4, "hardware": "GB300", - "quant": "NVFP4", "model": "zai-org/GLM-5.1", - "cell_id": "physical-arm"}, + bundle, + camp, + overrides={ + "tensor_parallel": 4, + "hardware": "GB300", + "quant": "NVFP4", + "model": "zai-org/GLM-5.1", + "cell_id": "physical-arm", + }, ) logical = { - "cell_id": "physical-arm-Kengine", "model": "zai-org/GLM-5.1", "hardware": "GB300", - "quant": "NVFP4", "tensor_parallel": 4, "parallel_strategy": "TP", - "mtp": False, "max_num_batched_tokens": 12288, "concurrency": 64, - "status": "full", "ttft_avg_ms": 1.0, "request_throughput_avg": 1.0, - "output_tps_per_gpu": 1.0, "backend": "vllm-sweep", + "cell_id": "physical-arm-Kengine", + "model": "zai-org/GLM-5.1", + "hardware": "GB300", + "quant": "NVFP4", + "tensor_parallel": 4, + "parallel_strategy": "TP", + "mtp": False, + "max_num_batched_tokens": 12288, + "concurrency": 64, + "status": "full", + "ttft_avg_ms": 1.0, + "request_throughput_avg": 1.0, + "output_tps_per_gpu": 1.0, + "backend": "vllm-sweep", "extra": {"arm": "physical-arm"}, } logical_dir = camp / "cells" / "logical" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_settle.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_settle.py index ef997e9..f5a3517 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_settle.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_settle.py @@ -58,6 +58,7 @@ def test_build_prewarm_command_unknown_shape(): def test_prewarm_returns_false_on_failure(): def runner(cmd, **kw): return subprocess.CompletedProcess(cmd, 1, stdout="", stderr="boom") + assert prewarm("http://x:8000", "m", ["aa-1k"], kube_wrap=lambda c: c, subprocess_runner=runner) is False diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline.py index 2275e88..857e15c 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline.py @@ -217,9 +217,16 @@ def test_load_ceilings_top_level_not_mapping_raises(tmp_path): def _row(hardware: str) -> AtlasCell: return AtlasCell( - cell_id="c", model="m", hardware=hardware, quant="NVFP4", - tensor_parallel=8, parallel_strategy="TP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status=STATUS_FULL, + cell_id="c", + model="m", + hardware=hardware, + quant="NVFP4", + tensor_parallel=8, + parallel_strategy="TP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status=STATUS_FULL, ) @@ -432,6 +439,7 @@ def test_render_page_raises_when_cell_kernels_empty(tmp_path): fig = plt.figure(figsize=(8, 11)) try: from collections import OrderedDict + with pytest.raises(ValueError, match="cell_kernels is empty"): sol_roofline.render_page( fig, diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline_scatter.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline_scatter.py index 73ea5da..2fd4294 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline_scatter.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_sol_roofline_scatter.py @@ -41,7 +41,6 @@ ) from tools.perf_tune_report.schema import BACKEND_VLLM_SWEEP, STATUS_FULL, AtlasCell - # --------------------------------------------------------------------------- # Fixtures (re-derived from test_sol_roofline.py to avoid coupling) # --------------------------------------------------------------------------- @@ -201,6 +200,7 @@ def test_peak_bandwidth_falls_back_to_hbm3(): def test_render_page_raises_when_empty(tmp_path): import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -214,6 +214,7 @@ def test_render_page_raises_when_empty(tmp_path): def test_render_page_raises_when_hw_key_missing(tmp_path): import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -394,16 +395,18 @@ def test_renderer_full_stack_5_pages_with_zymtrace_and_ncu(tmp_path, monkeypatch # Both zymtrace kernels.json AND ncu_kernels.json under same cell dir. cell_dir = campaign / "cells" / "cell-1" cell_dir.mkdir(parents=True) - (cell_dir / "kernels.json").write_text(json.dumps({ - "schema_version": 1, - "captured_sources": ["zymtrace"], - "top_kernels": [ - {"name": "multimem_all_reduce_kernel", "samples": 1000, "category": "NCCL"} - ], - "per_gpu": [{"gpu_name": "B200", "gpu_uuid": "u1", "samples": 1000}], - "per_category": {"NCCL": 500, "BMM-NVFP4": 500}, - "top_python_during_cuda": [{"frame": "f", "samples": 1}], - })) + (cell_dir / "kernels.json").write_text( + json.dumps( + { + "schema_version": 1, + "captured_sources": ["zymtrace"], + "top_kernels": [{"name": "multimem_all_reduce_kernel", "samples": 1000, "category": "NCCL"}], + "per_gpu": [{"gpu_name": "B200", "gpu_uuid": "u1", "samples": 1000}], + "per_category": {"NCCL": 500, "BMM-NVFP4": 500}, + "top_python_during_cuda": [{"frame": "f", "samples": 1}], + } + ) + ) (cell_dir / "ncu_kernels.json").write_text(json.dumps(_NCU_PAYLOAD)) atlas = _mk_atlas(campaign, "cell-1", hardware="B200") @@ -477,6 +480,7 @@ def test_renderer_full_stack_5_pages_with_zymtrace_and_ncu(tmp_path, monkeypatch def test_render_page_dcgm_fallback_plots_when_ncu_all_null(): """When all ncu kernels have null AI/tflops, DCGM fallback fires.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -503,6 +507,7 @@ def test_render_page_dcgm_fallback_plots_when_ncu_all_null(): def test_render_page_empty_state_message_when_no_data(): """When neither ncu kernels nor DCGM fallback have data, render an empty state.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -530,6 +535,7 @@ def test_render_page_empty_state_message_when_no_data(): def test_render_page_dcgm_fallback_skipped_when_attribution_empty(): """A DCGM payload without per_category_attribution renders an empty state.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -585,6 +591,7 @@ def test_render_page_sol_only_kernel_plotted_at_category_ceiling(): empty-state message, AI never fabricated. """ import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -603,12 +610,7 @@ def test_render_page_sol_only_kernel_plotted_at_category_ceiling(): axes_with_scatter = [a for a in fig.axes if a.collections] assert axes_with_scatter, "expected the SoL-only point to be plotted" # Exactly one point, at y ~= 0.746 * 2250 = 1678.5 TFLOPS. - ys = [ - float(c.get_offsets()[0][1]) - for a in axes_with_scatter - for c in a.collections - if len(c.get_offsets()) - ] + ys = [float(c.get_offsets()[0][1]) for a in axes_with_scatter for c in a.collections if len(c.get_offsets())] assert any(abs(y - 1678.5) < 1.0 for y in ys), f"expected ~1678.5 TFLOPS, got {ys}" all_text = " ".join(t.get_text() for ax in fig.axes for t in ax.texts) # Honest dynamic title + loud banner with how-to-fix. @@ -624,14 +626,13 @@ def test_render_page_sol_only_kernel_plotted_at_category_ceiling(): def test_render_page_status_full_for_real_ncu(): """Real ncu AI points -> partial=False + the 'byte+FLOP measured' title.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt fig = plt.figure(figsize=(8, 11)) try: - status = sol_roofline_scatter.render_page( - fig, OrderedDict([("c1", _NCU_PAYLOAD)]), _MIN_CEILINGS, "b200_sm100" - ) + status = sol_roofline_scatter.render_page(fig, OrderedDict([("c1", _NCU_PAYLOAD)]), _MIN_CEILINGS, "b200_sm100") assert status.partial is False assert status.reason == "" all_text = " ".join(t.get_text() for ax in fig.axes for t in ax.texts) @@ -644,14 +645,18 @@ def test_render_page_status_full_for_real_ncu(): def test_render_page_status_empty_is_partial(): """All-null ncu + no DCGM -> partial=True, reason ncu_scatter_empty.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt fig = plt.figure(figsize=(8, 11)) try: status = sol_roofline_scatter.render_page( - fig, OrderedDict([("c1", _NCU_PAYLOAD_ALL_NULL)]), _MIN_CEILINGS, - "b200_sm100", cell_dcgm=None, + fig, + OrderedDict([("c1", _NCU_PAYLOAD_ALL_NULL)]), + _MIN_CEILINGS, + "b200_sm100", + cell_dcgm=None, ) assert status.partial is True assert status.reason == "ncu_scatter_empty" @@ -662,6 +667,7 @@ def test_render_page_status_empty_is_partial(): def test_render_page_status_dcgm_fallback_not_partial(): """A measured DCGM workload-level fallback is NOT partial.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt @@ -681,13 +687,9 @@ def test_render_page_status_dcgm_fallback_not_partial(): def test_sol_only_helper_returns_none_for_bandwidth_category(): """Bandwidth-bound categories have no compute ceiling for the SoL-only y.""" - assert sol_roofline_scatter._category_compute_ceiling_tflops( - "FMHA", _MIN_CEILINGS, "b200_sm100" - ) is None + assert sol_roofline_scatter._category_compute_ceiling_tflops("FMHA", _MIN_CEILINGS, "b200_sm100") is None # Compute-bound Triton-fused -> bf16 2.25 PFLOPS -> 2250 TFLOPS. - assert sol_roofline_scatter._category_compute_ceiling_tflops( - "Triton-fused", _MIN_CEILINGS, "b200_sm100" - ) == 2250.0 + assert sol_roofline_scatter._category_compute_ceiling_tflops("Triton-fused", _MIN_CEILINGS, "b200_sm100") == 2250.0 def test_report_status_records_partial_page5_solonly(tmp_path, monkeypatch): @@ -716,6 +718,7 @@ def test_report_status_records_partial_page5_solonly(tmp_path, monkeypatch): def test_render_page_real_ncu_kernels_skip_dcgm_fallback(): """When ncu kernels have real AI/tflops, do not use the DCGM fallback.""" import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_spec_metrics.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_spec_metrics.py index c6d4cce..5a7c6b5 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_spec_metrics.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_spec_metrics.py @@ -20,7 +20,7 @@ compute_spec_window, parse_spec_totals, ) -from tools.perf_tune_report.schema import AtlasCell, STATUS_FULL +from tools.perf_tune_report.schema import STATUS_FULL, AtlasCell def _scrape_text(drafts: float, draft_tokens: float, accepted: float, *, pos0: float = 0.0) -> str: @@ -171,9 +171,7 @@ def __call__(self, cmd, capture_output=True, text=True, check=False): return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") if "python" in cmd and "-c" in cmd: # the /metrics scrape out = self.scrape_texts.pop(0) if self.scrape_texts else "" - return subprocess.CompletedProcess( - cmd, self.scrape_rc, stdout=out, stderr="scrape boom" - ) + return subprocess.CompletedProcess(cmd, self.scrape_rc, stdout=out, stderr="scrape boom") return subprocess.CompletedProcess(cmd, 0, stdout="bench ok", stderr="") diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_shape_label.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_shape_label.py index ade55f9..5656151 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_shape_label.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_shape_label.py @@ -5,6 +5,7 @@ ISL/OSL label (which would hide per-point variation). _shape_caption emits a single ISL/OSL only when the groups share it; otherwise "ISL/OSL: per-row (varies)". """ + from __future__ import annotations import sys diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_summary.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_summary.py index 7ffe60f..a273907 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_summary.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_tpm_summary.py @@ -178,9 +178,7 @@ def test_rows_without_output_tps_are_skipped(): def test_discover_tpm_config_reads_block(tmp_path): - (tmp_path / "config.yaml").write_text( - "name: c\ntpm:\n ttft_sla_ms: 500\n tpot_sla_ms: 50\n gpus_per_node: 4\n" - ) + (tmp_path / "config.yaml").write_text("name: c\ntpm:\n ttft_sla_ms: 500\n tpot_sla_ms: 50\n gpus_per_node: 4\n") cfg = discover_tpm_config(tmp_path) assert cfg.ttft_sla_ms == 500.0 assert cfg.tpot_sla_ms == 50.0 @@ -220,14 +218,12 @@ def test_discover_tpm_config_partial_block_fills_defaults(tmp_path): def test_discover_tpm_config_cost_block_overlays_defaults(tmp_path): # A cost: block overrides one default rate AND adds a brand-new hardware # key; the other default rates remain. - (tmp_path / "config.yaml").write_text( - "cost:\n usd_per_gpu_hour:\n B200: 5.0\n MI325X: 3.0\n" - ) + (tmp_path / "config.yaml").write_text("cost:\n usd_per_gpu_hour:\n B200: 5.0\n MI325X: 3.0\n") cfg = discover_tpm_config(tmp_path) - assert cfg.usd_per_gpu_hour["B200"] == 5.0 # override of default - assert cfg.usd_per_gpu_hour["MI325X"] == 3.0 # brand-new key added - assert cfg.usd_per_gpu_hour["H100"] == 6.16 # default retained - assert cfg.usd_per_gpu_hour["GB300"] == 12.00 # default estimate retained + assert cfg.usd_per_gpu_hour["B200"] == 5.0 # override of default + assert cfg.usd_per_gpu_hour["MI325X"] == 3.0 # brand-new key added + assert cfg.usd_per_gpu_hour["H100"] == 6.16 # default retained + assert cfg.usd_per_gpu_hour["GB300"] == 12.00 # default estimate retained assert "campaign config cost: block" in cfg.cost_rate_source @@ -259,10 +255,10 @@ def test_discover_tpm_config_overlays_fleet_cost_yaml(tmp_path): "usd_per_gpu_hour:\n GB300: 8.60\n B200: 6.50\n default: 8.60\n", ) cfg = discover_tpm_config(camp) - assert cfg.usd_per_gpu_hour["GB300"] == 8.60 # cost.yaml overrides the 12.00 default - assert cfg.usd_per_gpu_hour["B200"] == 6.50 # cost.yaml overrides the 8.60 default - assert cfg.usd_per_gpu_hour["H200"] == 6.31 # default retained (absent from cost.yaml) - assert "default" not in cfg.usd_per_gpu_hour # the fallback key is not a hardware + assert cfg.usd_per_gpu_hour["GB300"] == 8.60 # cost.yaml overrides the 12.00 default + assert cfg.usd_per_gpu_hour["B200"] == 6.50 # cost.yaml overrides the 8.60 default + assert cfg.usd_per_gpu_hour["H200"] == 6.31 # default retained (absent from cost.yaml) + assert "default" not in cfg.usd_per_gpu_hour # the fallback key is not a hardware assert "cost.yaml" in cfg.cost_rate_source @@ -274,15 +270,20 @@ def test_discover_tpm_config_campaign_block_wins_over_cost_yaml(tmp_path): config_text="cost:\n usd_per_gpu_hour:\n GB300: 4.00\n", ) cfg = discover_tpm_config(camp) - assert cfg.usd_per_gpu_hour["GB300"] == 4.00 # campaign block wins + assert cfg.usd_per_gpu_hour["GB300"] == 4.00 # campaign block wins assert "campaign config cost: block" in cfg.cost_rate_source def test_group_carries_mean_isl_osl_and_cache_mode(): rows = [ _row(concurrency=1, mean_input_tokens=3200.0, mean_output_tokens=512.0, cache_mode="warm"), - _row(concurrency=8, output_tps_per_gpu=30.0, mean_input_tokens=3200.0, - mean_output_tokens=512.0, cache_mode="warm"), + _row( + concurrency=8, + output_tps_per_gpu=30.0, + mean_input_tokens=3200.0, + mean_output_tokens=512.0, + cache_mode="warm", + ), ] s = compute_tpm_summary(rows) g = s.groups[0] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trend_view.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trend_view.py index 894586c..9312ca5 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trend_view.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trend_view.py @@ -14,17 +14,32 @@ from tools.perf_tune_report.perf_tune_report_cli import main from tools.perf_tune_report.schema import AtlasCell, write_jsonl -from tools.perf_tune_report.trend_view import build_trends, read_lake_rows, render_markdown +from tools.perf_tune_report.trend_view import build_trends, read_lake_rows def _row(captured_at: str, tps: float, **kw) -> AtlasCell: base = dict( - cell_id="c", model="GLM-5.1-NVFP4", hardware="GB300", quant="NVFP4", - tensor_parallel=4, parallel_strategy="TP", mtp=False, max_num_batched_tokens=8192, - concurrency=64, status="full", output_tps_per_gpu=tps, tpot_median_ms=20.0, - ttft_avg_ms=100.0, request_throughput_avg=1.0, cache_mode="cold", - dataset="random", cudagraph_mode="full", kv_cache_dtype="fp8_e4m3", - image="infr/vllm:v2.12.3", captured_at=captured_at, backend="vllm-sweep", + cell_id="c", + model="GLM-5.1-NVFP4", + hardware="GB300", + quant="NVFP4", + tensor_parallel=4, + parallel_strategy="TP", + mtp=False, + max_num_batched_tokens=8192, + concurrency=64, + status="full", + output_tps_per_gpu=tps, + tpot_median_ms=20.0, + ttft_avg_ms=100.0, + request_throughput_avg=1.0, + cache_mode="cold", + dataset="random", + cudagraph_mode="full", + kv_cache_dtype="fp8_e4m3", + image="infr/vllm:v2.12.3", + captured_at=captured_at, + backend="vllm-sweep", ) base.update(kw) return AtlasCell(**base) @@ -71,13 +86,27 @@ def test_lower_better_metric_regression_direction(): def test_cli_trend_view(tmp_path: Path, capsys): camps = tmp_path / "campaigns" - for name, ts, tps in [("a-20260601T000000Z", "2026-06-01T00:00:00Z", 1000.0), - ("b-20260607T000000Z", "2026-06-07T00:00:00Z", 700.0)]: + for name, ts, tps in [ + ("a-20260601T000000Z", "2026-06-01T00:00:00Z", 1000.0), + ("b-20260607T000000Z", "2026-06-07T00:00:00Z", 700.0), + ]: d = camps / name d.mkdir(parents=True) write_jsonl([_row(ts, tps)], d / "atlas.jsonl") - rc = main(["trend_view", "--campaigns-dir", str(camps), "--metric", "output_tps_per_gpu", - "--concurrency", "64", "--out", str(tmp_path / "TREND.md"), "--json"]) + rc = main( + [ + "trend_view", + "--campaigns-dir", + str(camps), + "--metric", + "output_tps_per_gpu", + "--concurrency", + "64", + "--out", + str(tmp_path / "TREND.md"), + "--json", + ] + ) assert rc == 0 env = json.loads(capsys.readouterr().out) assert env["n_trends"] == 1 and env["n_regressions"] == 1 @@ -91,7 +120,7 @@ def test_read_lake_rows_joins_vllm_commit_and_trends(tmp_path: Path): group into ONE trend, and the engine-version axis shows the joined vllm_commit, not the image tag (the A3 outcome).""" pq = pytest.importorskip("pyarrow.parquet") - import pyarrow as pa # noqa: WPS433 - test-only + import pyarrow as pa from tools.perf_tune_report.lake_writer import build_atlas_table @@ -106,16 +135,15 @@ def test_read_lake_rows_joins_vllm_commit_and_trends(tmp_path: Path): pq.write_table(build_atlas_table([_row(ts, tps)], camp), adir / "part-0.parquet") cdir = tmp_path / f"perflake/perf-report/campaign_v1/dt=2026-06-07/campaign={camp}" cdir.mkdir(parents=True) - pq.write_table(pa.table({"campaign_id": [camp], "vllm_commit": [commit]}), - cdir / "part-0.parquet") + pq.write_table(pa.table({"campaign_id": [camp], "vllm_commit": [commit]}), cdir / "part-0.parquet") rows = read_lake_rows(tmp_path) assert len(rows) == 2 view = build_trends(rows, metric="output_tps_per_gpu", regression_pct=10.0) - assert view["n_trends"] == 1 # same variant_key -> ONE trend across engine versions + assert view["n_trends"] == 1 # same variant_key -> ONE trend across engine versions t = view["trends"][0] assert t["n_points"] == 2 - assert t["regression"] is True # 1000 -> 750 tok/s + assert t["regression"] is True # 1000 -> 750 tok/s assert set(t["images"]) == {"aaaaaaa1", "bbbbbbb2"} # vllm_commit, not image tag @@ -125,8 +153,7 @@ def test_cli_trend_view_lake_dir(tmp_path: Path, capsys): adir = tmp_path / "perflake/perf-report/atlas_v1/dt=2026-06-07/campaign=camp-a" adir.mkdir(parents=True) - pq.write_table(build_atlas_table([_row("2026-06-01T00:00:00Z", 1000.0)], "camp-a"), - adir / "part-0.parquet") + pq.write_table(build_atlas_table([_row("2026-06-01T00:00:00Z", 1000.0)], "camp-a"), adir / "part-0.parquet") rc = main(["trend_view", "--lake-dir", str(tmp_path), "--json"]) assert rc == 0 env = json.loads(capsys.readouterr().out) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trtllm_stub.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trtllm_stub.py index cfd12aa..91b38ff 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trtllm_stub.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_trtllm_stub.py @@ -30,9 +30,16 @@ def test_backend_trtllm_in_backends_set(): def test_atlas_cell_accepts_trtllm_backend(): """AtlasCell schema validation must accept 'trtllm' as a backend value.""" cell = AtlasCell( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=1024, concurrency=1, status="full", + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=1024, + concurrency=1, + status="full", backend=BACKEND_TRTLLM, ) assert cell.backend == "trtllm" @@ -40,29 +47,49 @@ def test_atlas_cell_accepts_trtllm_backend(): def test_run_cell_raises_not_implemented(tmp_path): cfg = CellConfig( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=1024, concurrencies=(1,), + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=1024, + concurrencies=(1,), ) with pytest.raises(NotImplementedError, match="TRT-LLM backend is a stub"): run_cell( - cfg, tmp_path, - namespace="ns", bench_pod="pod", kube_context="ctx", - endpoint_url="http://x", served_model="m", + cfg, + tmp_path, + namespace="ns", + bench_pod="pod", + kube_context="ctx", + endpoint_url="http://x", + served_model="m", ) def test_run_cell_error_message_cites_contract(): cfg = CellConfig( - cell_id="x", model="m", hardware="H100", quant="FP8", - tensor_parallel=8, parallel_strategy="EP", mtp=False, - max_num_batched_tokens=1024, concurrencies=(1,), + cell_id="x", + model="m", + hardware="H100", + quant="FP8", + tensor_parallel=8, + parallel_strategy="EP", + mtp=False, + max_num_batched_tokens=1024, + concurrencies=(1,), ) try: run_cell( - cfg, Path("/tmp"), - namespace="ns", bench_pod="pod", kube_context="ctx", - endpoint_url="http://x", served_model="m", + cfg, + Path("/tmp"), + namespace="ns", + bench_pod="pod", + kube_context="ctx", + endpoint_url="http://x", + served_model="m", ) except NotImplementedError as exc: msg = str(exc) diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_value_view.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_value_view.py index 2b496c8..f41496c 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_value_view.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_value_view.py @@ -23,19 +23,37 @@ ) -def _mk_campaign(campaigns: Path, cid: str, *, sol_rigor: str = "L3", - dcgm: bool = True, tier: str = "verdict", - baseline_named: bool = True) -> Path: +def _mk_campaign( + campaigns: Path, + cid: str, + *, + sol_rigor: str = "L3", + dcgm: bool = True, + tier: str = "verdict", + baseline_named: bool = True, +) -> Path: d = campaigns / cid d.mkdir(parents=True) - (d / "report_status.json").write_text(json.dumps({ - "sol_rigor": sol_rigor, "sol_complete": True, - "dcgm_grounded": dcgm, "focus": "mixed", - })) - (d / "verdict.json").write_text(json.dumps({ - "tier": tier, "trials": 3, "same_node": True, - "baseline_named": baseline_named, - })) + (d / "report_status.json").write_text( + json.dumps( + { + "sol_rigor": sol_rigor, + "sol_complete": True, + "dcgm_grounded": dcgm, + "focus": "mixed", + } + ) + ) + (d / "verdict.json").write_text( + json.dumps( + { + "tier": tier, + "trials": 3, + "same_node": True, + "baseline_named": baseline_named, + } + ) + ) return d @@ -43,19 +61,46 @@ def _registry() -> dict: return { "baseline_context": "vLLM + FlashInfer-TRTLLM", "findings": [ - {"id": "win1", "title": "Win One", "lifecycle": "done", "baseline": "fp8", - "win": "1.3x", "hardware": "GB300", "deploy_readiness": "deployed", - "campaign_ids": ["camp-good"], "next_lever": "extend to fleet", "next_value": "high", - "source_refs": [{"repo": "example/vllm", "branch": "feature/x", - "commit": "abc1234", "delivery": "overlay"}]}, - {"id": "wip1", "title": "WIP One", "lifecycle": "in_progress", "baseline": "ctrl", - "win": "tbd", "hardware": "GB300", "deploy_readiness": "fork-local", - "campaign_ids": ["camp-missing"], "next_lever": "finish revalidation", "next_value": "med", - "source_refs": [{"repo": "vllm-project/vllm", "delivery": "image", - "image": "infr/vllm:v2.12.3"}]}, - {"id": "neg1", "title": "Neg One", "lifecycle": "closed_negative", "baseline": "FI", - "win": "slower", "hardware": "GB300", "deploy_readiness": "n/a", - "campaign_ids": [], "next_lever": "frontier-exhausted: H1 cannot beat H4", "next_value": "low"}, + { + "id": "win1", + "title": "Win One", + "lifecycle": "done", + "baseline": "fp8", + "win": "1.3x", + "hardware": "GB300", + "deploy_readiness": "deployed", + "campaign_ids": ["camp-good"], + "next_lever": "extend to fleet", + "next_value": "high", + "source_refs": [ + {"repo": "example/vllm", "branch": "feature/x", "commit": "abc1234", "delivery": "overlay"} + ], + }, + { + "id": "wip1", + "title": "WIP One", + "lifecycle": "in_progress", + "baseline": "ctrl", + "win": "tbd", + "hardware": "GB300", + "deploy_readiness": "fork-local", + "campaign_ids": ["camp-missing"], + "next_lever": "finish revalidation", + "next_value": "med", + "source_refs": [{"repo": "vllm-project/vllm", "delivery": "image", "image": "infr/vllm:v2.12.3"}], + }, + { + "id": "neg1", + "title": "Neg One", + "lifecycle": "closed_negative", + "baseline": "FI", + "win": "slower", + "hardware": "GB300", + "deploy_readiness": "n/a", + "campaign_ids": [], + "next_lever": "frontier-exhausted: H1 cannot beat H4", + "next_value": "low", + }, ], } @@ -80,11 +125,24 @@ def test_ungrounded_and_unnamed_flags(tmp_path): campaigns = tmp_path / "campaigns" campaigns.mkdir() _mk_campaign(campaigns, "camp-weak", sol_rigor="none", dcgm=False, baseline_named=False) - reg = {"baseline_context": "x", "findings": [ - {"id": "w", "title": "W", "lifecycle": "done", "baseline": "b", "win": "1x", - "hardware": "GB300", "deploy_readiness": "deployed", "campaign_ids": ["camp-weak"], - "next_lever": "next", "next_value": "med", "source_refs": [{"repo": "x/vllm", "delivery": "overlay"}]}, - ]} + reg = { + "baseline_context": "x", + "findings": [ + { + "id": "w", + "title": "W", + "lifecycle": "done", + "baseline": "b", + "win": "1x", + "hardware": "GB300", + "deploy_readiness": "deployed", + "campaign_ids": ["camp-weak"], + "next_lever": "next", + "next_value": "med", + "source_refs": [{"repo": "x/vllm", "delivery": "overlay"}], + }, + ], + } view = build_value_view(reg, campaigns) flags = view["findings"][0]["live"]["flags"] assert any("ungrounded" in f for f in flags) @@ -110,10 +168,21 @@ def test_missing_source_refs_flagged(tmp_path): campaigns = tmp_path / "campaigns" campaigns.mkdir() _mk_campaign(campaigns, "camp-good") - reg = {"baseline_context": "x", "findings": [ - {"id": "w", "title": "W", "lifecycle": "done", "baseline": "b", "win": "1x", - "hardware": "GB300", "deploy_readiness": "deployed", "campaign_ids": ["camp-good"]}, - ]} + reg = { + "baseline_context": "x", + "findings": [ + { + "id": "w", + "title": "W", + "lifecycle": "done", + "baseline": "b", + "win": "1x", + "hardware": "GB300", + "deploy_readiness": "deployed", + "campaign_ids": ["camp-good"], + }, + ], + } view = build_value_view(reg, campaigns) assert any("no source_refs" in f for f in view["findings"][0]["live"]["flags"]) @@ -131,10 +200,21 @@ def test_missing_next_lever_flagged(tmp_path): campaigns = tmp_path / "campaigns" campaigns.mkdir() _mk_campaign(campaigns, "camp-good") - reg = {"baseline_context": "x", "findings": [ - {"id": "w", "title": "W", "lifecycle": "closed_negative", "baseline": "b", "win": "1x", - "hardware": "GB300", "deploy_readiness": "n/a", "campaign_ids": []}, - ]} + reg = { + "baseline_context": "x", + "findings": [ + { + "id": "w", + "title": "W", + "lifecycle": "closed_negative", + "baseline": "b", + "win": "1x", + "hardware": "GB300", + "deploy_readiness": "n/a", + "campaign_ids": [], + }, + ], + } view = build_value_view(reg, campaigns) assert any("no next_lever" in f for f in view["findings"][0]["live"]["flags"]) @@ -164,8 +244,7 @@ def test_cli_value_view_report(tmp_path, capsys): _mk_campaign(campaigns, "camp-good") reg = tmp_path / "value-findings.yaml" reg.write_text(yaml.safe_dump(_registry())) - rc = main(["value_view", "--registry", str(reg), - "--campaigns-dir", str(campaigns), "--format", "report"]) + rc = main(["value_view", "--registry", str(reg), "--campaigns-dir", str(campaigns), "--format", "report"]) assert rc == 0 out = capsys.readouterr().out assert "value prop" in out and "**Win One**" in out @@ -178,13 +257,10 @@ def test_cli_value_view_resolves_gpu_hr_from_cost_yaml(tmp_path, capsys): campaigns.mkdir() _mk_campaign(campaigns, "camp-good") (tmp_path / "configs").mkdir() - (tmp_path / "configs" / "cost.yaml").write_text( - "usd_per_gpu_hour:\n GB300: 7.77\n default: 7.77\n" - ) + (tmp_path / "configs" / "cost.yaml").write_text("usd_per_gpu_hour:\n GB300: 7.77\n default: 7.77\n") reg = tmp_path / "value-findings.yaml" reg.write_text(yaml.safe_dump(_registry())) - rc = main(["value_view", "--registry", str(reg), - "--campaigns-dir", str(campaigns), "--json"]) + rc = main(["value_view", "--registry", str(reg), "--campaigns-dir", str(campaigns), "--json"]) assert rc == 0 payload = json.loads(capsys.readouterr().out) assert payload["gpu_hr"] == 7.77 # from cost.yaml, not the 8.60 GB300 default @@ -198,8 +274,7 @@ def test_cli_value_view_gpu_hr_override_wins_over_cost_yaml(tmp_path, capsys): (tmp_path / "configs" / "cost.yaml").write_text("usd_per_gpu_hour:\n GB300: 7.77\n") reg = tmp_path / "value-findings.yaml" reg.write_text(yaml.safe_dump(_registry())) - rc = main(["value_view", "--registry", str(reg), "--campaigns-dir", str(campaigns), - "--gpu-hr", "12.34", "--json"]) + rc = main(["value_view", "--registry", str(reg), "--campaigns-dir", str(campaigns), "--gpu-hr", "12.34", "--json"]) assert rc == 0 payload = json.loads(capsys.readouterr().out) assert payload["gpu_hr"] == 12.34 # explicit override wins @@ -209,19 +284,52 @@ def _gain_registry() -> dict: """Registry exercising the blended gain ranking: a throughput speedup with tok/s/GPU (-> $ columns), a latency speedup without tok/s/GPU, and a high-next_value lever w/ no gain.""" src = [{"repo": "x/vllm", "delivery": "overlay"}] - return {"baseline_context": "x", "findings": [ - {"id": "thr_big", "title": "Thr Big", "lifecycle": "done", "baseline": "b", "win": "2.3x", - "hardware": "GB300", "deploy_readiness": "deployed", "campaign_ids": [], - "next_lever": "roll to fleet", "next_value": "med", "source_refs": src, - "gain": {"speedup": 2.3, "tier": "throughput", "tps_gpu_peak": 481, "baseline_tps_gpu": 369}}, - {"id": "lat_mid", "title": "Lat Mid", "lifecycle": "done", "baseline": "b", "win": "2.1x", - "hardware": "GB300", "deploy_readiness": "deployed", "campaign_ids": [], - "next_lever": "port to fleet", "next_value": "low", "source_refs": src, - "gain": {"speedup": 2.1, "tier": "latency"}}, - {"id": "no_gain_high", "title": "No Gain High", "lifecycle": "in_progress", "baseline": "b", - "win": "tbd", "hardware": "GB300", "deploy_readiness": "fork-local", "campaign_ids": [], - "next_lever": "infra fix", "next_value": "high", "source_refs": src}, - ]} + return { + "baseline_context": "x", + "findings": [ + { + "id": "thr_big", + "title": "Thr Big", + "lifecycle": "done", + "baseline": "b", + "win": "2.3x", + "hardware": "GB300", + "deploy_readiness": "deployed", + "campaign_ids": [], + "next_lever": "roll to fleet", + "next_value": "med", + "source_refs": src, + "gain": {"speedup": 2.3, "tier": "throughput", "tps_gpu_peak": 481, "baseline_tps_gpu": 369}, + }, + { + "id": "lat_mid", + "title": "Lat Mid", + "lifecycle": "done", + "baseline": "b", + "win": "2.1x", + "hardware": "GB300", + "deploy_readiness": "deployed", + "campaign_ids": [], + "next_lever": "port to fleet", + "next_value": "low", + "source_refs": src, + "gain": {"speedup": 2.1, "tier": "latency"}, + }, + { + "id": "no_gain_high", + "title": "No Gain High", + "lifecycle": "in_progress", + "baseline": "b", + "win": "tbd", + "hardware": "GB300", + "deploy_readiness": "fork-local", + "campaign_ids": [], + "next_lever": "infra fix", + "next_value": "high", + "source_refs": src, + }, + ], + } def test_frontier_ranks_by_gain_then_next_value(tmp_path): @@ -258,13 +366,25 @@ def test_evidence_ids_render_supporting_count_no_flags(tmp_path): campaigns = tmp_path / "campaigns" campaigns.mkdir() _mk_campaign(campaigns, "camp-good") - reg = {"baseline_context": "x", "findings": [ - {"id": "w", "title": "W", "lifecycle": "done", "baseline": "b", "win": "1x", - "hardware": "GB300", "deploy_readiness": "deployed", "campaign_ids": ["camp-good"], - "next_lever": "next", "next_value": "med", - "source_refs": [{"repo": "x/vllm", "delivery": "overlay"}], - "evidence_ids": ["e1-20260101T000000Z", "e2-20260101T000001Z"]}, - ]} + reg = { + "baseline_context": "x", + "findings": [ + { + "id": "w", + "title": "W", + "lifecycle": "done", + "baseline": "b", + "win": "1x", + "hardware": "GB300", + "deploy_readiness": "deployed", + "campaign_ids": ["camp-good"], + "next_lever": "next", + "next_value": "med", + "source_refs": [{"repo": "x/vllm", "delivery": "overlay"}], + "evidence_ids": ["e1-20260101T000000Z", "e2-20260101T000001Z"], + }, + ], + } view = build_value_view(reg, campaigns) # evidence_ids add NO flags (the clean-ledger design goal: only campaign_ids carry flags) assert view["findings"][0]["live"]["flags"] == [] diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_verdict_gate.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_verdict_gate.py index a349cb0..7ad61e0 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_verdict_gate.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_verdict_gate.py @@ -60,8 +60,12 @@ def test_verdict_problems_verdict_incomplete_lists_each_gap(): def test_verdict_problems_latency_claim_requires_tpot_or_itl(): v = VerdictSummary( - tier="verdict", trials=3, same_node=True, baseline_named=True, - latency_claim=True, decode_metric="throughput", + tier="verdict", + trials=3, + same_node=True, + baseline_named=True, + latency_claim=True, + decode_metric="throughput", ) assert any("decode_metric in tpot|itl" in x for x in verdict_problems(v)) v.decode_metric = "tpot" @@ -70,8 +74,12 @@ def test_verdict_problems_latency_claim_requires_tpot_or_itl(): def test_verdict_problems_which_kernel_requires_per_kernel_ref(): v = VerdictSummary( - tier="verdict", trials=3, same_node=True, baseline_named=True, - which_kernel_claim=True, per_kernel_ref=False, + tier="verdict", + trials=3, + same_node=True, + baseline_named=True, + which_kernel_claim=True, + per_kernel_ref=False, ) assert any("per_kernel_ref" in x for x in verdict_problems(v)) @@ -104,8 +112,13 @@ def test_publish_verdict_incomplete_strict_raises(tmp_path: Path): def test_publish_verdict_complete_passes(tmp_path: Path): cd = _stage_campaign(tmp_path) _write_verdict( - cd, tier="verdict", trials=3, same_node=True, baseline_named=True, - latency_claim=True, decode_metric="tpot", + cd, + tier="verdict", + trials=3, + same_node=True, + baseline_named=True, + latency_claim=True, + decode_metric="tpot", ) publish(cd, cfg=_stub_cfg(), dry_run=True) # must not raise @@ -119,8 +132,13 @@ def test_publish_verdict_incomplete_allow_incomplete_lands_as_draft(tmp_path: Pa def test_campaign_row_records_verdict_tier(tmp_path: Path): cd = _stage_campaign(tmp_path) _write_verdict( - cd, tier="verdict", trials=3, same_node=True, baseline_named=True, - latency_claim=True, decode_metric="tpot", + cd, + tier="verdict", + trials=3, + same_node=True, + baseline_named=True, + latency_claim=True, + decode_metric="tpot", ) table = build_campaign_row(cd, [_make_atlas_row()]) assert table.column("verdict_tier")[0].as_py() == "verdict" diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_zymtrace_kernels.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_zymtrace_kernels.py index e1750b5..d97560a 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/test_zymtrace_kernels.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/test_zymtrace_kernels.py @@ -42,17 +42,12 @@ AtlasCell, ) - # --------------------------------------------------------------------------- # Synthetic-bundle helpers # --------------------------------------------------------------------------- _VALID_TSVS = { - "kernel-class.tsv": ( - "event_kind\tkind\tsamples\n" - "cuda\tnative\t1000\n" - "cuda\tcuda\t500\n" - ), + "kernel-class.tsv": ("event_kind\tkind\tsamples\ncuda\tnative\t1000\ncuda\tcuda\t500\n"), "top-gpu-frames.tsv": ( "kernel\tsamples\n" "multimem_all_reduce_kernel\t558\n" @@ -67,12 +62,7 @@ "NVIDIA B200\t92f929f2-1fea-72bb-7d96-834eb0f4e9c8\t35917\n" ), "per-category.tsv": ( - "category\tsamples\n" - "cuBLAS\t119199\n" - "Triton-fused\t61950\n" - "NCCL\t16510\n" - "BMM-NVFP4\t19212\n" - "FMHA\t9226\n" + "category\tsamples\ncuBLAS\t119199\nTriton-fused\t61950\nNCCL\t16510\nBMM-NVFP4\t19212\nFMHA\t9226\n" ), "top-python-during-cuda.tsv": ( "python_frame\tsamples\n" @@ -163,15 +153,9 @@ def test_zymtrace_positive_path(tmp_path): assert "zymtrace" in payload["captured_sources"] assert payload["per_category"]["cuBLAS"] == 119199 # category derivation: multimem_all_reduce_kernel -> NCCL - assert any( - k["name"].startswith("multimem_all_reduce") and k["category"] == "NCCL" - for k in payload["top_kernels"] - ) + assert any(k["name"].startswith("multimem_all_reduce") and k["category"] == "NCCL" for k in payload["top_kernels"]) # category derivation: bmm_E2m1E2m1 -> BMM-NVFP4 - assert any( - k["name"].startswith("bmm_E2m1") and k["category"] == "BMM-NVFP4" - for k in payload["top_kernels"] - ) + assert any(k["name"].startswith("bmm_E2m1") and k["category"] == "BMM-NVFP4" for k in payload["top_kernels"]) def test_zymtrace_declared_but_empty_tsv_raises(tmp_path): diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/tpm_summary.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/tpm_summary.py index e7a8a6f..027ae65 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/tpm_summary.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/tpm_summary.py @@ -42,10 +42,14 @@ import csv import io import json +import logging import sys +from collections.abc import Sequence from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Any, Sequence +from typing import Any + +import yaml from tools.perf_tune_report.schema import AtlasCell @@ -69,7 +73,10 @@ # (~1.4x B200), a placeholder to replace when a real rate exists. All are LIST/ # ESTIMATE prices, not contract cost; override per-campaign. DEFAULT_USD_PER_GPU_HOUR: dict[str, float] = { - "H100": 6.16, "H200": 6.31, "B200": 8.60, "GB300": 12.00, + "H100": 6.16, + "H200": 6.31, + "B200": 8.60, + "GB300": 12.00, } DEFAULT_COST_RATE_SOURCE = ( "assumed: H100/H200/B200 = representative public on-demand list rate per-GPU " @@ -83,6 +90,7 @@ OPERATING_POINT_SLA = "sla" BASES = ("per_gpu", "per_replica", "per_node") +logger = logging.getLogger(__name__) _METHODOLOGY_CAVEAT = ( "TPM = tokens/minute (tok/s * 60). 'peak' is the warm sweep best-case " @@ -109,9 +117,7 @@ class TpmConfig: # {hardware: $/GPU-hour}. Defaults to the assumed public-list table # (DEFAULT_USD_PER_GPU_HOUR); a campaign ``cost:`` block is overlaid on top # (so a campaign can override a rate or add a hardware key like GB300). - usd_per_gpu_hour: dict[str, float] = field( - default_factory=lambda: dict(DEFAULT_USD_PER_GPU_HOUR) - ) + usd_per_gpu_hour: dict[str, float] = field(default_factory=lambda: dict(DEFAULT_USD_PER_GPU_HOUR)) # Provenance for the cost rates in use (informational; surfaced in the # summary, NOT a lake column). Set to the campaign config when a cost: block # overrides the defaults. @@ -148,10 +154,9 @@ def discover_tpm_config(campaign_dir: Path) -> TpmConfig: if not cfg_path.is_file(): return TpmConfig() try: - import yaml # lazy: keep the module import light for the publish path - data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {} - except Exception: # noqa: BLE001 - config is best-effort; degrade to defaults + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Ignoring malformed TPM config %s: %s", cfg_path, exc) return TpmConfig() if not isinstance(data, dict): return TpmConfig() @@ -186,22 +191,15 @@ def _num(key: str, default: float) -> float: if fleet: usd_map.update(fleet) source = f"perf-tune-report/configs/cost.yaml (rates {sorted(fleet)})" - except Exception: # noqa: BLE001 - cost.yaml is best-effort; degrade to defaults - pass + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + logger.debug("Ignoring malformed cost config %s: %s", cost_yaml, exc) if isinstance(cost_block, dict): raw_map = cost_block.get("usd_per_gpu_hour") if isinstance(raw_map, dict): - overrides = { - str(hw): float(v) - for hw, v in raw_map.items() - if isinstance(v, (int, float)) and v > 0 - } + overrides = {str(hw): float(v) for hw, v in raw_map.items() if isinstance(v, (int, float)) and v > 0} if overrides: usd_map.update(overrides) - source = ( - f"campaign config cost: block (overrides {sorted(overrides)}) " - f"on top of {source}" - ) + source = f"campaign config cost: block (overrides {sorted(overrides)}) on top of {source}" return TpmConfig( ttft_sla_ms=_num("ttft_sla_ms", DEFAULT_TTFT_SLA_MS), @@ -266,10 +264,7 @@ class TpmGroup: @property def legend_label(self) -> str: mtp_suffix = " MTP" if self.mtp else "" - return ( - f"{self.hardware} {self.quant}{mtp_suffix} " - f"TP={self.tensor_parallel} {self.parallel_strategy}" - ) + return f"{self.hardware} {self.quant}{mtp_suffix} TP={self.tensor_parallel} {self.parallel_strategy}" def to_dict(self) -> dict[str, Any]: d = asdict(self) @@ -320,12 +315,25 @@ def to_csv(self) -> str: writer = csv.writer(buf) writer.writerow( [ - "model", "hardware", "quant", "tensor_parallel", - "parallel_strategy", "mtp", "operating_point", "basis", - "concurrency", "output_tpm", "total_tpm", - "ttft_avg_ms", "tpot_median_ms", "itl_avg_ms", - "mean_isl", "mean_osl", "cache_mode", - "usd_per_1m_output_tokens", "usd_per_1m_total_tokens", + "model", + "hardware", + "quant", + "tensor_parallel", + "parallel_strategy", + "mtp", + "operating_point", + "basis", + "concurrency", + "output_tpm", + "total_tpm", + "ttft_avg_ms", + "tpot_median_ms", + "itl_avg_ms", + "mean_isl", + "mean_osl", + "cache_mode", + "usd_per_1m_output_tokens", + "usd_per_1m_total_tokens", ] ) for g in self.groups: @@ -335,15 +343,23 @@ def to_csv(self) -> str: for basis in BASES: writer.writerow( [ - g.model, g.hardware, g.quant, g.tensor_parallel, - g.parallel_strategy, g.mtp, point.operating_point, - basis, point.concurrency, + g.model, + g.hardware, + g.quant, + g.tensor_parallel, + g.parallel_strategy, + g.mtp, + point.operating_point, + basis, + point.concurrency, _fmt_num(getattr(point, f"output_tpm_{basis}")), _fmt_num(getattr(point, f"total_tpm_{basis}")), _fmt_num(point.ttft_avg_ms), _fmt_num(point.tpot_median_ms), _fmt_num(point.itl_avg_ms), - _fmt_num(g.mean_isl), _fmt_num(g.mean_osl), g.cache_mode, + _fmt_num(g.mean_isl), + _fmt_num(g.mean_osl), + g.cache_mode, _fmt_num(point.usd_per_1m_output_tokens), _fmt_num(point.usd_per_1m_total_tokens), ] @@ -360,10 +376,7 @@ def to_markdown(self) -> str: lines.append(f"- Data source / shape: {self.context_line}") lines.append(f"- gpus_per_node basis: {self.gpus_per_node}") if self.sla_computed: - lines.append( - f"- SLA thresholds: TTFT <= {self.ttft_sla_ms} ms, " - f"TPOT/ITL <= {self.tpot_sla_ms} ms" - ) + lines.append(f"- SLA thresholds: TTFT <= {self.ttft_sla_ms} ms, TPOT/ITL <= {self.tpot_sla_ms} ms") else: lines.append( "- SLA thresholds: not set -> SLA columns not computed " @@ -411,10 +424,7 @@ def to_markdown(self) -> str: elif self.sla_computed: # SLA thresholds were set but no sweep point met them: # show an explicit row so the gap is never silently absent. - lines.append( - f"| {g.legend_label} | sla | - | no point met SLA " - "| - | - | - | - | - | - | - |" - ) + lines.append(f"| {g.legend_label} | sla | - | no point met SLA | - | - | - | - | - | - | - |") lines.append("") return "\n".join(lines) @@ -452,9 +462,7 @@ def _ordered_unique(items: Sequence[str]) -> list[str]: return list(seen.keys()) -def _meets_sla( - row: AtlasCell, ttft_sla_ms: float | None, tpot_sla_ms: float | None -) -> bool: +def _meets_sla(row: AtlasCell, ttft_sla_ms: float | None, tpot_sla_ms: float | None) -> bool: """True iff the row satisfies every supplied latency threshold. A threshold of ``None`` is not applied. When a threshold is applied the @@ -477,23 +485,20 @@ def _point_from_row( gpus_per_node: int, usd_per_gpu_hour: float | None = None, ) -> TpmPoint: - o_gpu = row.output_tps_per_gpu * SECONDS_PER_MINUTE - t_gpu = ( - row.total_tps_per_gpu * SECONDS_PER_MINUTE - if row.total_tps_per_gpu is not None - else None - ) + output_tps_per_gpu = _required_output_tps(row) + o_gpu = output_tps_per_gpu * SECONDS_PER_MINUTE + t_gpu = row.total_tps_per_gpu * SECONDS_PER_MINUTE if row.total_tps_per_gpu is not None else None # $/1M tokens (basis-independent: cost and tokens both scale with GPU count, # so the per-GPU ratio is the $/token for any basis). usd_out = usd_tot = None - if usd_per_gpu_hour is not None and row.output_tps_per_gpu > 0: - usd_out = usd_per_gpu_hour * 1e6 / (row.output_tps_per_gpu * 3600.0) + if usd_per_gpu_hour is not None and output_tps_per_gpu > 0: + usd_out = usd_per_gpu_hour * 1e6 / (output_tps_per_gpu * 3600.0) if row.total_tps_per_gpu and row.total_tps_per_gpu > 0: usd_tot = usd_per_gpu_hour * 1e6 / (row.total_tps_per_gpu * 3600.0) return TpmPoint( operating_point=operating_point, concurrency=row.concurrency, - output_tps_per_gpu=row.output_tps_per_gpu, + output_tps_per_gpu=output_tps_per_gpu, total_tps_per_gpu=row.total_tps_per_gpu, output_tpm_per_gpu=o_gpu, output_tpm_per_replica=o_gpu * tensor_parallel, @@ -511,6 +516,14 @@ def _point_from_row( ) +def _required_output_tps(row: AtlasCell) -> float: + """Return throughput after the summary's row filter has validated it.""" + output_tps_per_gpu = row.output_tps_per_gpu + if output_tps_per_gpu is None: + raise ValueError("TPM operating points require output_tps_per_gpu") + return output_tps_per_gpu + + def _group_attr(group_rows: Sequence[AtlasCell], attr: str) -> float | None: """The single non-None value of a per-group-constant numeric attr (e.g. mean_isl/mean_osl), or the mean if rows disagree. None when all absent.""" @@ -546,7 +559,7 @@ def compute_tpm_summary( for row in rows: if row.output_tps_per_gpu is None: continue - key = (row.model,) + row.legend_key + key = (row.model, *row.legend_key) groups.setdefault(key, []).append(row) out_groups: list[TpmGroup] = [] @@ -562,11 +575,7 @@ def compute_tpm_summary( # publish. Only a NON-default (operator-supplied) unmatched key is a typo. if cost_map: present_hw = {key[1] for key in groups} - unmatched = [ - hw - for hw in cost_map - if hw not in present_hw and hw not in DEFAULT_USD_PER_GPU_HOUR - ] + unmatched = [hw for hw in cost_map if hw not in present_hw and hw not in DEFAULT_USD_PER_GPU_HOUR] if unmatched: print( f"WARNING: cost: usd_per_gpu_hour has hardware key(s) {unmatched} " @@ -575,24 +584,18 @@ def compute_tpm_summary( "'b200' vs 'B200').", file=sys.stderr, ) - for key, group_rows in groups.items(): - peak_row = max(group_rows, key=lambda r: r.output_tps_per_gpu) + for _key, group_rows in groups.items(): + peak_row = max(group_rows, key=_required_output_tps) tp = peak_row.tensor_parallel usd_gpu_hr = cost_map.get(peak_row.hardware) - peak = _point_from_row( - peak_row, OPERATING_POINT_PEAK, tp, gpus_per_node, usd_gpu_hr - ) + peak = _point_from_row(peak_row, OPERATING_POINT_PEAK, tp, gpus_per_node, usd_gpu_hr) sla_point: TpmPoint | None = None if sla_active: - sla_candidates = [ - r for r in group_rows if _meets_sla(r, ttft_sla_ms, tpot_sla_ms) - ] + sla_candidates = [r for r in group_rows if _meets_sla(r, ttft_sla_ms, tpot_sla_ms)] if sla_candidates: - sla_row = max(sla_candidates, key=lambda r: r.output_tps_per_gpu) - sla_point = _point_from_row( - sla_row, OPERATING_POINT_SLA, tp, gpus_per_node, usd_gpu_hr - ) + sla_row = max(sla_candidates, key=_required_output_tps) + sla_point = _point_from_row(sla_row, OPERATING_POINT_SLA, tp, gpus_per_node, usd_gpu_hr) out_groups.append( TpmGroup( diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/trend_view.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/trend_view.py index afa77a9..469c3e3 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/trend_view.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/trend_view.py @@ -11,6 +11,7 @@ grouping applies to a published-lake parquet pull (lake rows carry ``variant_key`` + ``captured_at`` + ``image`` directly), so this is the local-first longitudinal view. """ + from __future__ import annotations from pathlib import Path @@ -78,7 +79,8 @@ def read_lake_rows(lake_dir: Path, *, hardware_filter: str | None = None) -> lis if hardware_filter and rec.get("hardware") != hardware_filter: continue rec = dict(rec) - commit = commit_by_campaign.get(rec.get("campaign_id"), "") + campaign_id = rec.get("campaign_id") + commit = commit_by_campaign.get(campaign_id, "") if isinstance(campaign_id, str) else "" if commit: # prefer the joined commit for the engine-version axis rec["image"] = commit rows.append(SimpleNamespace(**rec)) @@ -110,11 +112,13 @@ def build_trends( if not vk: continue key = (canon_model(r.model), vk, int(r.concurrency)) - groups.setdefault(key, []).append({ - "captured_at": r.captured_at or "", - "value": float(val), - "image": getattr(r, "image", "") or "", - }) + groups.setdefault(key, []).append( + { + "captured_at": r.captured_at or "", + "value": float(val), + "image": getattr(r, "image", "") or "", + } + ) trends: list[dict[str, Any]] = [] for (model, vk, conc), pts in groups.items(): @@ -125,20 +129,22 @@ def build_trends( if len(pts) >= 2 and first["value"]: delta_pct = round((last["value"] / first["value"] - 1) * 100, 1) regression = (delta_pct >= regression_pct) if lower_better else (delta_pct <= -regression_pct) - trends.append({ - "model": model, - "variant_key": vk[:12], - "concurrency": conc, - "n_points": len(pts), - "first_value": first["value"], - "last_value": last["value"], - "delta_pct": delta_pct, - "regression": regression, - "first_captured_at": first["captured_at"], - "last_captured_at": last["captured_at"], - "images": sorted({p["image"] for p in pts if p["image"]}), - "points": pts, - }) + trends.append( + { + "model": model, + "variant_key": vk[:12], + "concurrency": conc, + "n_points": len(pts), + "first_value": first["value"], + "last_value": last["value"], + "delta_pct": delta_pct, + "regression": regression, + "first_captured_at": first["captured_at"], + "last_captured_at": last["captured_at"], + "images": sorted({p["image"] for p in pts if p["image"]}), + "points": pts, + } + ) # Regressions first, then most-tracked (most points) first, then model. trends.sort(key=lambda t: (not t["regression"], -t["n_points"], t["model"])) return { @@ -174,7 +180,8 @@ def render_markdown(view: dict[str, Any], *, title: str = "Perf/quality trend ov dpct = f"{t['delta_pct']:+.1f}%" if t["delta_pct"] is not None else "-" window = ( f"{t['first_captured_at'][:10]}..{t['last_captured_at'][:10]}" - if t["n_points"] >= 2 else (t["last_captured_at"][:10] or "-") + if t["n_points"] >= 2 + else (t["last_captured_at"][:10] or "-") ) imgs = ", ".join(i.split(":")[-1] for i in t["images"]) or "-" lines.append( diff --git a/plugins/profile-and-optimize/server/tools/perf_tune_report/value_view.py b/plugins/profile-and-optimize/server/tools/perf_tune_report/value_view.py index 68fa3c0..84a9f62 100644 --- a/plugins/profile-and-optimize/server/tools/perf_tune_report/value_view.py +++ b/plugins/profile-and-optimize/server/tools/perf_tune_report/value_view.py @@ -19,6 +19,7 @@ Dependency-light by design: reads only small JSON files (no pyarrow / matplotlib), so it runs in a minimal install. Added in profile-and-optimize (perf_tune_report value_view verb). """ + from __future__ import annotations import json @@ -118,16 +119,23 @@ def build_value_view(registry: dict, campaigns_root: Path) -> dict[str, Any]: best_rigor = _max_rigor(best_rigor, st["sol_rigor"]) if not st["dcgm_grounded"] and st["sol_rigor"] in ("none", "L1"): flags.append(f"{cid}: ungrounded (sol_rigor={st['sol_rigor']})") - if f.get("finding_baseline_check", True) and not st["baseline_named"] \ - and f.get("lifecycle") in ("done", "in_progress"): + if ( + f.get("finding_baseline_check", True) + and not st["baseline_named"] + and f.get("lifecycle") in ("done", "in_progress") + ): flags.append(f"{cid}: baseline_named=false (tag verdict.json)") # Code-under-test provenance match (rigor principle p): the finding's # source_refs delivery+commit MUST match the cited campaign's provenance -- # an overlay/offline-prepped campaign cited as an infr-patch benefit is a # cross-tier DRAFT defect (even when the kernels match). - flags.extend(_prov.provenance_match_problems( - f.get("source_refs"), _read_campaign_provenance(cdir), label=cid, - )) + flags.extend( + _prov.provenance_match_problems( + f.get("source_refs"), + _read_campaign_provenance(cdir), + label=cid, + ) + ) if cids and not any_found: flags.append("no backing campaign found locally") if f.get("lifecycle") in ("done", "in_progress") and not cids: @@ -142,10 +150,12 @@ def build_value_view(registry: dict, campaigns_root: Path) -> dict[str, Any]: # a closed negative points at the next lever. Flag a missing one. if not str(f.get("next_lever", "") or "").strip(): flags.append("no next_lever (performance ratchet: name the next lever or 'frontier-exhausted: ')") - out_findings.append({ - **f, - "live": {"best_sol_rigor": best_rigor, "campaigns": camps, "flags": flags}, - }) + out_findings.append( + { + **f, + "live": {"best_sol_rigor": best_rigor, "campaigns": camps, "flags": flags}, + } + ) return { "baseline_context": registry.get("baseline_context", ""), "findings": out_findings, @@ -183,17 +193,14 @@ def _count_by_lifecycle(view: dict) -> dict[str, int]: return counts -def render_markdown(view: dict, *, title: str = "Value ledger", - gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> str: +def render_markdown(view: dict, *, title: str = "Value ledger", gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> str: counts = _count_by_lifecycle(view) lines: list[str] = [ f"# {title} (generated by `perftunereport value_view`)", "", f"**Baseline:** {view['baseline_context']}", "", - "Summary: " + " | ".join( - f"{lc}={counts.get(lc, 0)}" for lc, _ in LIFECYCLE_GROUPS - ), + "Summary: " + " | ".join(f"{lc}={counts.get(lc, 0)}" for lc, _ in LIFECYCLE_GROUPS), "", "> Win numbers are human-verified in the registry; the `live sol_rigor / verdict`" " + `Flags` columns are read from the perf-lake campaigns at render time.", @@ -218,14 +225,12 @@ def render_markdown(view: dict, *, title: str = "Value ledger", n_ev = len(f.get("evidence_ids") or []) if n_ev: camps += f" (+{n_ev} supporting)" - tiers = ", ".join(sorted({ - c.get("verdict_tier", "?") for c in live["campaigns"] if c.get("found") - })) or "-" + tiers = ", ".join(sorted({c.get("verdict_tier", "?") for c in live["campaigns"] if c.get("found")})) or "-" flags = "; ".join(live["flags"]) or "ok" lines.append( - f"| {f.get('title','')} | {f.get('win','')} | {f.get('baseline','')} | " - f"{f.get('hardware','')} | {live['best_sol_rigor']} / {tiers} | " - f"{f.get('deploy_readiness','')} | {_source_summary(f)} | {camps} | {flags} |" + f"| {f.get('title', '')} | {f.get('win', '')} | {f.get('baseline', '')} | " + f"{f.get('hardware', '')} | {live['best_sol_rigor']} / {tiers} | " + f"{f.get('deploy_readiness', '')} | {_source_summary(f)} | {camps} | {flags} |" ) lines.append("") lines.append(render_frontier(view, gpu_hr=gpu_hr)) @@ -257,14 +262,11 @@ def _finding_economics(f: dict, gpu_hr: float) -> dict[str, Any]: base_tps = g.get("baseline_tps_gpu") dollars_peak = _fleet._cost(tps_peak, gpu_hr) if isinstance(tps_peak, (int, float)) else None dollars_base = _fleet._cost(base_tps, gpu_hr) if isinstance(base_tps, (int, float)) else None - dollars_saved = ( - dollars_base - dollars_peak - if (dollars_peak is not None and dollars_base is not None) else None - ) + dollars_saved = dollars_base - dollars_peak if (dollars_peak is not None and dollars_base is not None) else None gpu_hours_saved = ( (1.0 / 3600.0) * 1e6 * (1.0 / base_tps - 1.0 / tps_peak) - if isinstance(tps_peak, (int, float)) and isinstance(base_tps, (int, float)) - and tps_peak and base_tps else None + if isinstance(tps_peak, (int, float)) and isinstance(base_tps, (int, float)) and tps_peak and base_tps + else None ) if isinstance(speedup, (int, float)): gain_label = f"{speedup:g}x {_TIER_TAG.get(tier, tier or 'x')}".strip() @@ -293,22 +295,26 @@ def frontier_rows(view: dict, *, gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> list[ if not nl: continue econ = _finding_economics(f, gpu_hr) - rows.append({ - "id": f.get("id", ""), - "title": f.get("title", ""), - "lifecycle": f.get("lifecycle", "?"), - "next_lever": nl, - "next_value": str(f.get("next_value", "med") or "med").lower(), - "exhausted": nl.lower().startswith("frontier-exhausted"), - **econ, - }) - rows.sort(key=lambda r: ( - r["exhausted"], - 0 if r["ranked_by_speedup"] else 1, - -(r["speedup"] or 0.0) if r["ranked_by_speedup"] else 0.0, - _NEXT_VALUE_ORDER.get(r["next_value"], 1), - _LIFECYCLE_RANK.get(r["lifecycle"], 9), - )) + rows.append( + { + "id": f.get("id", ""), + "title": f.get("title", ""), + "lifecycle": f.get("lifecycle", "?"), + "next_lever": nl, + "next_value": str(f.get("next_value", "med") or "med").lower(), + "exhausted": nl.lower().startswith("frontier-exhausted"), + **econ, + } + ) + rows.sort( + key=lambda r: ( + r["exhausted"], + 0 if r["ranked_by_speedup"] else 1, + -(r["speedup"] or 0.0) if r["ranked_by_speedup"] else 0.0, + _NEXT_VALUE_ORDER.get(r["next_value"], 1), + _LIFECYCLE_RANK.get(r["lifecycle"], 9), + ) + ) return rows @@ -320,8 +326,7 @@ def render_frontier(view: dict, *, gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> str rows = frontier_rows(view, gpu_hr=gpu_hr) active = [r for r in rows if not r["exhausted"]] exhausted = [r for r in rows if r["exhausted"]] - missing = [f.get("id", "?") for f in view["findings"] - if not str(f.get("next_lever", "") or "").strip()] + missing = [f.get("id", "?") for f in view["findings"] if not str(f.get("next_lever", "") or "").strip()] lines: list[str] = [ "## GRIND FRONTIER -- ranked next levers (performance ratchet)", "", @@ -332,11 +337,15 @@ def render_frontier(view: dict, *, gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> str "", ] if not active: - lines.append("> WARNING: no ACTIVE next levers queued -- the frontier looks exhausted. " - "Verify every finding's `next_lever` (the ratchet forbids a stall).") + lines.append( + "> WARNING: no ACTIVE next levers queued -- the frontier looks exhausted. " + "Verify every finding's `next_lever` (the ratchet forbids a stall)." + ) lines.append("") else: - lines.append("| # | gain | $/1M out (peak) | $ saved/1M | GPU-hrs saved/1M | Next lever | From finding | Lifecycle |") + lines.append( + "| # | gain | $/1M out (peak) | $ saved/1M | GPU-hrs saved/1M | Next lever | From finding | Lifecycle |" + ) lines.append("|---|---|---|---|---|---|---|---|") for i, r in enumerate(active, 1): dpm = f"${r['dollars_per_1m']:.2f}" if r["dollars_per_1m"] is not None else "-" @@ -348,12 +357,16 @@ def render_frontier(view: dict, *, gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> str ) lines.append("") if missing: - lines.append(f"> RATCHET FLAG: {len(missing)} finding(s) missing `next_lever`: " - + ", ".join(missing) + " -- name the next lever.") + lines.append( + f"> RATCHET FLAG: {len(missing)} finding(s) missing `next_lever`: " + + ", ".join(missing) + + " -- name the next lever." + ) lines.append("") if exhausted: - lines.append(f"_Banked / frontier-exhausted ({len(exhausted)}): " - + ", ".join(r["id"] for r in exhausted) + "._") + lines.append( + f"_Banked / frontier-exhausted ({len(exhausted)}): " + ", ".join(r["id"] for r in exhausted) + "._" + ) lines.append("") return "\n".join(lines) @@ -366,15 +379,13 @@ def render_frontier(view: dict, *, gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> str } -def render_report(view: dict, *, title: str = "Inference perf wins -- value prop", - gpu_hr: float = _fleet.GPU_HR_DEFAULT) -> str: +def render_report( + view: dict, *, title: str = "Inference perf wins -- value prop", gpu_hr: float = _fleet.GPU_HR_DEFAULT +) -> str: """Compact, copy-paste-ready markdown for a report or Slack: a one-line summary + grouped bullets (one per finding), instead of the wide audit table.""" counts = _count_by_lifecycle(view) - summary = ", ".join( - f"{_GROUP_LABEL[lc].split(' -- ')[0]} {counts.get(lc, 0)}" - for lc, _ in LIFECYCLE_GROUPS - ) + summary = ", ".join(f"{_GROUP_LABEL[lc].split(' -- ')[0]} {counts.get(lc, 0)}" for lc, _ in LIFECYCLE_GROUPS) lines: list[str] = [ f"# {title}", f"_Baseline: {view['baseline_context']}_", @@ -392,17 +403,19 @@ def render_report(view: dict, *, title: str = "Inference perf wins -- value prop lines.append(f"**{_GROUP_LABEL[lc]} ({len(fs)})**") for f in fs: live = f["live"] - meta = [b for b in ( - f.get("baseline"), - f.get("hardware"), - f"sol_rigor {live['best_sol_rigor']}", - f.get("deploy_readiness"), - (f"src {_source_summary(f)}" if f.get("source_refs") else None), - (f"+{len(f['evidence_ids'])} supporting runs" if f.get("evidence_ids") else None), - ) if b] - lines.append( - f"- **{f.get('title', '')}** -- {f.get('win', '')} _({'; '.join(meta)})_" - ) + meta = [ + b + for b in ( + f.get("baseline"), + f.get("hardware"), + f"sol_rigor {live['best_sol_rigor']}", + f.get("deploy_readiness"), + (f"src {_source_summary(f)}" if f.get("source_refs") else None), + (f"+{len(f['evidence_ids'])} supporting runs" if f.get("evidence_ids") else None), + ) + if b + ] + lines.append(f"- **{f.get('title', '')}** -- {f.get('win', '')} _({'; '.join(meta)})_") lines.append("") lines.append(render_frontier(view, gpu_hr=gpu_hr)) flags = sorted({fl for f in view["findings"] for fl in f["live"]["flags"]}) diff --git a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/host_overhead.py b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/host_overhead.py index f0b0b24..d174153 100644 --- a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/host_overhead.py +++ b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/host_overhead.py @@ -90,7 +90,15 @@ def cmd_top(args: argparse.Namespace) -> int: def _record_flamegraph( - *, pyspy: str, pid: int, duration: int, rate: int, subprocesses: bool, gil: bool, idle: bool, out: Path, + *, + pyspy: str, + pid: int, + duration: int, + rate: int, + subprocesses: bool, + gil: bool, + idle: bool, + out: Path, ) -> int: out.parent.mkdir(parents=True, exist_ok=True) cmd = [ @@ -118,7 +126,16 @@ def _record_flamegraph( def _record_top_summary( - *, pyspy: str, pid: int, duration: int, rate: int, subprocesses: bool, gil: bool, idle: bool, out: Path, top_n: int, + *, + pyspy: str, + pid: int, + duration: int, + rate: int, + subprocesses: bool, + gil: bool, + idle: bool, + out: Path, + top_n: int, ) -> int: out.parent.mkdir(parents=True, exist_ok=True) speedscope = out.with_suffix(".speedscope.json") @@ -172,7 +189,7 @@ def _summarise_speedscope(path: Path, *, top_n: int) -> str: continue if len(weights) != len(samples): continue - for stack, weight in zip(samples, weights): + for stack, weight in zip(samples, weights, strict=True): if not stack: continue total_units += weight @@ -290,53 +307,61 @@ def populate_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: sub = parser.add_subparsers(dest="command", required=True) common_pid = argparse.ArgumentParser(add_help=False) - common_pid.add_argument("--pid", type=int, required=True, - help="PID of the rank-0 (or any) process to sample.") - common_pid.add_argument("--rate", type=int, default=DEFAULT_RATE_HZ, - help=f"Sampling rate in Hz. Default {DEFAULT_RATE_HZ}.") - common_pid.add_argument("--subprocesses", action="store_true", - help="Profile all subprocesses (e.g. dataloader workers).") - common_pid.add_argument("--gil", action="store_true", - help="Only sample threads holding the Python GIL.") - common_pid.add_argument("--idle", action="store_true", - help="Include idle/blocked threads in the sample stream.") + common_pid.add_argument("--pid", type=int, required=True, help="PID of the rank-0 (or any) process to sample.") + common_pid.add_argument( + "--rate", type=int, default=DEFAULT_RATE_HZ, help=f"Sampling rate in Hz. Default {DEFAULT_RATE_HZ}." + ) + common_pid.add_argument( + "--subprocesses", action="store_true", help="Profile all subprocesses (e.g. dataloader workers)." + ) + common_pid.add_argument("--gil", action="store_true", help="Only sample threads holding the Python GIL.") + common_pid.add_argument("--idle", action="store_true", help="Include idle/blocked threads in the sample stream.") top = sub.add_parser("top", parents=[common_pid], help="Live CPU-time-by-function display (Ctrl-C to exit).") - top.add_argument("--json", action="store_true", - help="No-op; py-spy top is interactive and prints to the terminal.") + top.add_argument("--json", action="store_true", help="No-op; py-spy top is interactive and prints to the terminal.") top.set_defaults(func=cmd_top) - record = sub.add_parser("record", parents=[common_pid], - help="Record stacks for --duration seconds; emit flamegraph + top-N summary.") - record.add_argument("--duration", type=int, default=DEFAULT_DURATION_SEC, - help=f"Sampling duration in seconds. Default {DEFAULT_DURATION_SEC}.") - record.add_argument("--top-n", type=int, default=DEFAULT_TOP_N, - help=f"Number of top frames in the text summary. Default {DEFAULT_TOP_N}.") - record.add_argument("--art-dir", type=Path, default=None, - help="Artifact dir; flame -> host-overhead-flame.svg; " - "text -> host-overhead.txt.") - record.add_argument("--flame-out", type=Path, default=None, - help="Override the flamegraph output path.") - record.add_argument("--text-out", type=Path, default=None, - help="Override the top-N text summary output path.") - record.add_argument("--json", action="store_true", - help="No-op; record emits the flamegraph SVG + top-N text summary.") + record = sub.add_parser( + "record", parents=[common_pid], help="Record stacks for --duration seconds; emit flamegraph + top-N summary." + ) + record.add_argument( + "--duration", + type=int, + default=DEFAULT_DURATION_SEC, + help=f"Sampling duration in seconds. Default {DEFAULT_DURATION_SEC}.", + ) + record.add_argument( + "--top-n", + type=int, + default=DEFAULT_TOP_N, + help=f"Number of top frames in the text summary. Default {DEFAULT_TOP_N}.", + ) + record.add_argument( + "--art-dir", + type=Path, + default=None, + help="Artifact dir; flame -> host-overhead-flame.svg; text -> host-overhead.txt.", + ) + record.add_argument("--flame-out", type=Path, default=None, help="Override the flamegraph output path.") + record.add_argument("--text-out", type=Path, default=None, help="Override the top-N text summary output path.") + record.add_argument( + "--json", action="store_true", help="No-op; record emits the flamegraph SVG + top-N text summary." + ) record.set_defaults(func=cmd_record) - dump = sub.add_parser("dump", parents=[common_pid], - help="One-shot stack snapshot (cheapest).") - dump.add_argument("--out", type=Path, default=None, - help="Write the dump here (default: stdout).") - dump.add_argument("--locals", action="store_true", - help="Show local variables in each frame.") - dump.add_argument("--json", action="store_true", - help="No-op; py-spy dump emits a stack snapshot to --out / stdout.") + dump = sub.add_parser("dump", parents=[common_pid], help="One-shot stack snapshot (cheapest).") + dump.add_argument("--out", type=Path, default=None, help="Write the dump here (default: stdout).") + dump.add_argument("--locals", action="store_true", help="Show local variables in each frame.") + dump.add_argument( + "--json", action="store_true", help="No-op; py-spy dump emits a stack snapshot to --out / stdout." + ) dump.set_defaults(func=cmd_dump) return parser def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + description = (__doc__ or "Sample a running process with py-spy").splitlines()[0] + parser = argparse.ArgumentParser(description=description) populate_parser(parser) return parser.parse_args(argv) diff --git a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_cli.py b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_cli.py index a59ba60..5d0742c 100644 --- a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_cli.py +++ b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_cli.py @@ -29,7 +29,6 @@ profile_diff, ) - CONTRACT: dict[str, dict[str, Any]] = { "host-overhead": { "safety": "writes_artifacts", @@ -43,9 +42,17 @@ "safety": "writes_artifacts", "required": (), "optional": ( - "--baseline", "--baseline-csv-dir", "--candidate", - "--candidate-csv-dir", "--baseline-label", "--candidate-label", - "--out", "--json-out", "--limit", "--scratch", "--json", + "--baseline", + "--baseline-csv-dir", + "--candidate", + "--candidate-csv-dir", + "--baseline-label", + "--candidate-label", + "--out", + "--json-out", + "--limit", + "--scratch", + "--json", ), "json": True, "ack": None, @@ -79,7 +86,8 @@ def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + description = (__doc__ or "Operator-facing profiling CLI").splitlines()[0] + parser = argparse.ArgumentParser(description=description) sub = parser.add_subparsers(dest="verb", required=True) for verb, description in _VERB_DESCRIPTIONS.items(): verb_parser = sub.add_parser(verb, description=description) diff --git a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_diff.py b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_diff.py index 1e7e25a..ccd18aa 100644 --- a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_diff.py +++ b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/profile_diff.py @@ -111,16 +111,13 @@ def run_nsys_stats(report_path: Path, out_dir: Path) -> dict[str, Path]: subprocess.run(cmd, check=True, capture_output=True, text=True) except subprocess.CalledProcessError as exc: raise RuntimeError( - f"nsys stats failed for {report_path}:\n" - f" stdout:\n{exc.stdout}\n stderr:\n{exc.stderr}" + f"nsys stats failed for {report_path}:\n stdout:\n{exc.stdout}\n stderr:\n{exc.stderr}" ) from exc paths = {} for report in NSYS_REPORTS: candidate = out_dir / f"{report_path.stem}_{report}.csv" if not candidate.is_file(): - raise RuntimeError( - f"nsys stats did not produce expected CSV: {candidate}" - ) + raise RuntimeError(f"nsys stats did not produce expected CSV: {candidate}") paths[report] = candidate return paths @@ -133,13 +130,9 @@ def collect_csv_dir(csv_dir: Path) -> dict[str, Path]: for report in NSYS_REPORTS: matches = sorted(csv_dir.glob(f"*_{report}.csv")) if not matches: - raise FileNotFoundError( - f"no CSV matching *_{report}.csv in {csv_dir}" - ) + raise FileNotFoundError(f"no CSV matching *_{report}.csv in {csv_dir}") if len(matches) > 1: - raise RuntimeError( - f"ambiguous CSV match for {report} in {csv_dir}: {matches}" - ) + raise RuntimeError(f"ambiguous CSV match for {report} in {csv_dir}: {matches}") paths[report] = matches[0] return paths @@ -193,10 +186,7 @@ def _resolve(cands: tuple[str, ...]) -> int | None: total_idx = _resolve(total_col_candidates) count_idx = _resolve(count_col_candidates) if count_col_candidates else None if name_idx is None or total_idx is None: - raise RuntimeError( - f"could not locate name/total columns in {csv_path}; " - f"header was {header_norm}" - ) + raise RuntimeError(f"could not locate name/total columns in {csv_path}; header was {header_norm}") for raw in reader: if not raw or len(raw) <= max(name_idx, total_idx): continue @@ -273,10 +263,7 @@ def _idx(cands: tuple[str, ...]) -> int | None: continue totals[name] += _to_int(raw[dur_idx]) counts[name] += 1 - return [ - StatRow(name=name, total_ns=totals[name], count=counts[name]) - for name in totals - ] + return [StatRow(name=name, total_ns=totals[name], count=counts[name]) for name in totals] def keep_interesting_nvtx(rows: list[StatRow]) -> list[StatRow]: @@ -401,8 +388,7 @@ def render_report( f"Baseline: `{baseline_label}` ", f"Candidate: `{candidate_label}`", "", - "Sorted by absolute delta-of-totals; top " - f"`{limit}` rows per table.", + f"Sorted by absolute delta-of-totals; top `{limit}` rows per table.", "", "Each row's `Delta` column is `candidate - baseline`. Positive numbers " "indicate the candidate spent MORE device time in that range / kernel " @@ -427,34 +413,36 @@ def populate_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: `profile_diff --json-out file.json`. """ src = parser.add_argument_group("inputs (mutually exclusive per side)") - src.add_argument("--baseline", type=Path, default=None, - help="Path to the baseline .nsys-rep.") - src.add_argument("--baseline-csv-dir", type=Path, default=None, - help="Pre-extracted nsys-stats directory for the baseline.") - src.add_argument("--candidate", type=Path, default=None, - help="Path to the candidate .nsys-rep.") - src.add_argument("--candidate-csv-dir", type=Path, default=None, - help="Pre-extracted nsys-stats directory for the candidate.") - parser.add_argument("--baseline-label", default=None, - help="Override the baseline label printed in the report.") - parser.add_argument("--candidate-label", default=None, - help="Override the candidate label printed in the report.") - parser.add_argument("--out", type=Path, default=None, - help="Write the report to this path. Default: stdout.") - parser.add_argument("--json-out", dest="json_out", type=Path, default=None, - help="Optional JSON sidecar with the full delta tables.") - parser.add_argument("--json", action="store_true", - help="No-op; --json-out PATH is the real JSON sidecar flag.") - parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT, - help=f"Top-N rows per table. Default {DEFAULT_LIMIT}.") - parser.add_argument("--scratch", type=Path, default=None, - help="Scratch directory for nsys-stats CSV extraction. " - "Default: /nsys-stats or a tempdir.") + src.add_argument("--baseline", type=Path, default=None, help="Path to the baseline .nsys-rep.") + src.add_argument( + "--baseline-csv-dir", type=Path, default=None, help="Pre-extracted nsys-stats directory for the baseline." + ) + src.add_argument("--candidate", type=Path, default=None, help="Path to the candidate .nsys-rep.") + src.add_argument( + "--candidate-csv-dir", type=Path, default=None, help="Pre-extracted nsys-stats directory for the candidate." + ) + parser.add_argument("--baseline-label", default=None, help="Override the baseline label printed in the report.") + parser.add_argument("--candidate-label", default=None, help="Override the candidate label printed in the report.") + parser.add_argument("--out", type=Path, default=None, help="Write the report to this path. Default: stdout.") + parser.add_argument( + "--json-out", dest="json_out", type=Path, default=None, help="Optional JSON sidecar with the full delta tables." + ) + parser.add_argument("--json", action="store_true", help="No-op; --json-out PATH is the real JSON sidecar flag.") + parser.add_argument( + "--limit", type=int, default=DEFAULT_LIMIT, help=f"Top-N rows per table. Default {DEFAULT_LIMIT}." + ) + parser.add_argument( + "--scratch", + type=Path, + default=None, + help="Scratch directory for nsys-stats CSV extraction. Default: /nsys-stats or a tempdir.", + ) return parser def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + description = (__doc__ or "Diff two Nsight Systems profiles").splitlines()[0] + parser = argparse.ArgumentParser(description=description) populate_parser(parser) return parser.parse_args(argv) diff --git a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/test_profile_diff.py b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/test_profile_diff.py index 836362e..6ccc564 100644 --- a/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/test_profile_diff.py +++ b/plugins/profile-and-optimize/server/tools/pipeline/submission/profile/test_profile_diff.py @@ -19,7 +19,7 @@ THIS_DIR = Path(__file__).resolve().parent sys.path.insert(0, str(THIS_DIR)) -import profile_diff # noqa: E402 +import profile_diff def _write(path: Path, text: str) -> Path: @@ -119,7 +119,7 @@ def test_parse_nvtx_strict_on_malformed(self) -> None: '"Time (%)","Total Time (ns)","Instances","Avg (ns)","Med (ns)",' '"Min (ns)","Max (ns)","StdDev (ns)","Range"\n' '50.0,1000000000,10,100000000,100000000,90000000,110000000,1000000,"good"\n' - '50.0,truncated_row\n', + "50.0,truncated_row\n", ) rows = profile_diff.parse_nvtx(csv) self.assertEqual([r.name for r in rows], ["good"]) @@ -259,13 +259,20 @@ def test_main_with_csv_dirs_emits_report_and_json(self) -> None: out_json = root / "out" / "profile-diff.json" rc = profile_diff.main( [ - "--baseline-csv-dir", str(base), - "--candidate-csv-dir", str(cand), - "--baseline-label", "B", - "--candidate-label", "C", - "--out", str(out_md), - "--json-out", str(out_json), - "--limit", "5", + "--baseline-csv-dir", + str(base), + "--candidate-csv-dir", + str(cand), + "--baseline-label", + "B", + "--candidate-label", + "C", + "--out", + str(out_md), + "--json-out", + str(out_json), + "--limit", + "5", ] ) self.assertEqual(rc, 0) diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/INSTALL.md b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/INSTALL.md index 916c411..30667ae 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/INSTALL.md +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/INSTALL.md @@ -10,10 +10,12 @@ can launch a command with environment variables can use the manual stdio form. - Python 3.11 or newer with `venv` support. - Bash and network access for the Python package install. - A clone of this repository. -- `rg` for the two read-only MCP search tools. - GPU, cluster, profiler, and credential access only for workflows that need them. The local install and MCP handshake do not need a GPU. +The two read-only MCP search tools prefer `rg` when it is available. They fall +back to `grep`, so `rg` is not required. + ## Install from the repository root Run one command from the root of this repository: diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/pyproject.toml b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/pyproject.toml index 0fc9dbc..170e3f4 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/pyproject.toml +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "profile-and-optimize-mcp" -version = "0.3.1" +version = "0.4.0" description = "Standards-based MCP runtime for GPU inference profiling and optimization tools." readme = "README.md" requires-python = ">=3.11" diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/configure_clients.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/configure_clients.py index abf814e..3660389 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/configure_clients.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/configure_clients.py @@ -95,7 +95,7 @@ def update_json_mcp( servers = data.setdefault("mcpServers", {}) if not isinstance(servers, dict): - raise ValueError(f"{path}: mcpServers must be a JSON object") # noqa: TRY004 + raise ValueError(f"{path}: mcpServers must be a JSON object") servers[SERVER_NAME] = server_block(args, client=client) full_text = json.dumps(data, indent=2, sort_keys=False) + "\n" @@ -245,8 +245,8 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--client", action="append", choices=(*CLIENTS, "all"), - default=[], - help="Client to configure. May be passed multiple times. Default: cursor.", + required=True, + help="Client to configure. May be passed multiple times.", ) parser.add_argument("--repo-root", type=Path, default=Path.cwd()) parser.add_argument( @@ -295,11 +295,7 @@ def _file_config_path(client: str, args: argparse.Namespace) -> Path: def configure_client(client: str, args: argparse.Namespace) -> None: - if ( - args.registration == "auto" - and client in CLI_CLIENTS - and try_official_cli(client, args) - ): + if args.registration == "auto" and client in CLI_CLIENTS and try_official_cli(client, args): return config_path = _file_config_path(client, args) @@ -311,7 +307,7 @@ def configure_client(client: str, args: argparse.Namespace) -> None: def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) - clients = args.client or ["cursor"] + clients = args.client if "all" in clients: clients = list(CLIENTS) else: diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/install_profile_and_optimize_mcp.sh b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/install_profile_and_optimize_mcp.sh index feec9d2..c94d833 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/install_profile_and_optimize_mcp.sh +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/scripts/install_profile_and_optimize_mcp.sh @@ -21,7 +21,7 @@ Installs the complete bundled server, then registers it with selected clients. Options: --client NAME cursor | claude | codex | gemini | antigravity | all - May be passed multiple times. Default: cursor. + May be passed multiple times. Required. --repo-root PATH Bundled server root. Default: auto-detected server directory. --venv PATH Python venv path. Default: ~/.local/share/profile-and-optimize-mcp-venv. --python PATH Python interpreter used to create the venv. Default: python3. @@ -115,6 +115,12 @@ if [[ "${REGISTRATION}" != "auto" && "${REGISTRATION}" != "file" ]]; then exit 2 fi +if [[ "${#CLIENT_ARGS[@]}" -eq 0 ]]; then + printf '%s\n' 'missing required --client. Choose claude or codex for a first-class install.' >&2 + usage >&2 + exit 2 +fi + if [[ ! -f "${SERVER_ROOT}/pyproject.toml" || ! -f "${SERVER_ROOT}/mcp_surface.py" || ! -d "${SERVER_ROOT}/tools/profile_and_optimize_mcp" || ! -f "${SERVER_ROOT}/install.sh" ]]; then printf 'FATAL: bundled server root is incomplete: %s\n' "${SERVER_ROOT}" >&2 printf 'Expected pyproject.toml, mcp_surface.py, install.sh, and tools/profile_and_optimize_mcp/.\n' >&2 @@ -127,9 +133,7 @@ CONFIG_COMMAND_ARGS=( --registration "${REGISTRATION}" "${CONFIG_ARGS[@]}" ) -if [[ "${#CLIENT_ARGS[@]}" -gt 0 ]]; then - CONFIG_COMMAND_ARGS+=("${CLIENT_ARGS[@]}") -fi +CONFIG_COMMAND_ARGS+=("${CLIENT_ARGS[@]}") if [[ "${DRY_RUN}" -eq 1 ]]; then printf '%s\n' '=== [dry-run] bundled server install ===' diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/__init__.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/__init__.py index b315ff4..7a01661 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/__init__.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/__init__.py @@ -1,3 +1,3 @@ """GPU profiling and performance optimization MCP server.""" -__version__ = "0.3.1" +__version__ = "0.4.0" diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/repo.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/repo.py index 7dc48d5..20b0ac6 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/repo.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/repo.py @@ -19,11 +19,7 @@ def find_repo_root(start: Path | None = None) -> Path: """Return the bundled server root using product files as markers.""" env_root = os.environ.get("PROFILE_AND_OPTIMIZE_REPO_ROOT") - current = ( - _operator_path(env_root, label="PROFILE_AND_OPTIMIZE_REPO_ROOT") - if env_root - else (start or Path.cwd()) - ) + current = _operator_path(env_root, label="PROFILE_AND_OPTIMIZE_REPO_ROOT") if env_root else (start or Path.cwd()) current = current.resolve() if current.is_file(): current = current.parent @@ -35,9 +31,7 @@ def find_repo_root(start: Path | None = None) -> Path: ): return current current = current.parent - raise RuntimeError( - "cannot locate the profile-and-optimize server root. Set PROFILE_AND_OPTIMIZE_REPO_ROOT" - ) + raise RuntimeError("cannot locate the profile-and-optimize server root. Set PROFILE_AND_OPTIMIZE_REPO_ROOT") def repo_path(*parts: str) -> Path: diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/server.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/server.py index 35c90ee..42b45b0 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/server.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src/profile_and_optimize_mcp/server.py @@ -200,9 +200,7 @@ def _dynamic_ack( raise TypeError(f"invalid dynamic acknowledgement declaration for {verb} {argv[0]}") for arg in argv[1:]: if _is_option_abbreviation(arg, required_with): - raise ValueError( - f"abbreviated option {arg!r} is not allowed; use {required_with!r}" - ) + raise ValueError(f"abbreviated option {arg!r} is not allowed; use {required_with!r}") return ack_flag, required_with in argv, str(safety) if safety else None @@ -220,9 +218,7 @@ def run_surface_tool(name: str, params: dict[str, Any] | None = None) -> dict[st ) effective_ack_flag = ack_flag or dynamic_ack_flag ack_exempt_when = module.CONTRACT[spec.verb].get("ack_exempt_when", ()) - if not isinstance(ack_exempt_when, (tuple, list)) or not all( - isinstance(flag, str) for flag in ack_exempt_when - ): + if not isinstance(ack_exempt_when, (tuple, list)) or not all(isinstance(flag, str) for flag in ack_exempt_when): raise TypeError(f"invalid acknowledgement exemption for {spec.library} {spec.verb}") static_ack_required = bool(ack_flag) and not any(flag in argv for flag in ack_exempt_when) ack_required = static_ack_required or dynamic_ack_required @@ -231,15 +227,12 @@ def run_surface_tool(name: str, params: dict[str, Any] | None = None) -> dict[st ( arg for arg in argv - if effective_ack_flag - and (arg == effective_ack_flag or _is_option_abbreviation(arg, effective_ack_flag)) + if effective_ack_flag and (arg == effective_ack_flag or _is_option_abbreviation(arg, effective_ack_flag)) ), None, ) if effective_ack_flag and raw_ack: - raise ValueError( - f"pass {ack_field}=true instead of including {raw_ack!r} in params.args" - ) + raise ValueError(f"pass {ack_field}=true instead of including {raw_ack!r} in params.args") if ack_required and ack_field and params.get(ack_field) is not True: raise PermissionError(f"{spec.name} requires {ack_field}=true") if ack_required and effective_ack_flag and ack_field: diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_configure_clients.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_configure_clients.py index 43ad883..fd39ab4 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_configure_clients.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_configure_clients.py @@ -39,7 +39,7 @@ class JsonConfigTests(unittest.TestCase): def test_antigravity_default_uses_official_global_path(self) -> None: home = Path("/tmp/test-antigravity-home") with mock.patch.object(configure_clients.Path, "home", return_value=home): - args = configure_clients.parse_args([]) + args = configure_clients.parse_args(["--client", "antigravity"]) self.assertEqual( args.antigravity_config, home / ".gemini" / "config" / "mcp_config.json", @@ -249,9 +249,7 @@ def test_generated_config_contains_only_runtime_environment(self) -> None: config = root / "config.toml" configure_clients.update_codex_toml(config, make_args(root)) - profile = tomllib.loads(config.read_text(encoding="utf-8"))[ - "mcp_servers" - ]["profile_and_optimize"] + profile = tomllib.loads(config.read_text(encoding="utf-8"))["mcp_servers"]["profile_and_optimize"] self.assertEqual( set(profile["env"]), {"PROFILE_AND_OPTIMIZE_REPO_ROOT"}, diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_installers.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_installers.py index 668cf70..52aadb8 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_installers.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_installers.py @@ -9,13 +9,7 @@ REPO_ROOT = Path(__file__).resolve().parents[6] SERVER_ROOT = REPO_ROOT / "plugins" / "profile-and-optimize" / "server" SERVER_INSTALLER = SERVER_ROOT / "install.sh" -MCP_INSTALLER = ( - SERVER_ROOT - / "tools" - / "profile_and_optimize_mcp" - / "scripts" - / "install_profile_and_optimize_mcp.sh" -) +MCP_INSTALLER = SERVER_ROOT / "tools" / "profile_and_optimize_mcp" / "scripts" / "install_profile_and_optimize_mcp.sh" AGENT_SKILL_INSTALLER = REPO_ROOT / "scripts" / "install-agent-skills.sh" CURSOR_SKILL_INSTALLER = REPO_ROOT / "scripts" / "install-skills-into-cursor.sh" SOURCE_SKILLS = REPO_ROOT / "plugins" / "profile-and-optimize" / "skills" @@ -241,11 +235,33 @@ def test_invalid_client_fails_before_installing(self) -> None: self.assertFalse(venv.exists()) self.assertIn("invalid client", result.stderr) + def test_mcp_installer_requires_an_explicit_client(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + venv = root / "venv" + + result = subprocess.run( + [ + "bash", + str(MCP_INSTALLER), + "--venv", + str(venv), + "--dry-run", + ], + cwd=REPO_ROOT, + env={**os.environ, "HOME": str(root / "home")}, + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(result.returncode, 2) + self.assertFalse(venv.exists()) + self.assertIn("missing required --client", result.stderr) + def test_agent_skill_installer_is_idempotent_for_codex_and_cursor(self) -> None: expected_names = { - path.name - for path in SOURCE_SKILLS.iterdir() - if path.is_dir() and (path / "SKILL.md").is_file() + path.name for path in SOURCE_SKILLS.iterdir() if path.is_dir() and (path / "SKILL.md").is_file() } for client in ("codex", "cursor"): with self.subTest(client=client), tempfile.TemporaryDirectory() as temporary_directory: @@ -287,9 +303,7 @@ def test_agent_skill_installer_is_idempotent_for_codex_and_cursor(self) -> None: def test_agent_skill_installer_preserves_conflicts_for_codex_and_cursor(self) -> None: skill_name = min( - path.name - for path in SOURCE_SKILLS.iterdir() - if path.is_dir() and (path / "SKILL.md").is_file() + path.name for path in SOURCE_SKILLS.iterdir() if path.is_dir() and (path / "SKILL.md").is_file() ) for client in ("codex", "cursor"): with self.subTest(client=client), tempfile.TemporaryDirectory() as temporary_directory: diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_plugin_manifest.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_plugin_manifest.py index cead78f..5f2943c 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_plugin_manifest.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_plugin_manifest.py @@ -4,7 +4,6 @@ import unittest from pathlib import Path - PLUGIN_ROOT = Path(__file__).resolve().parents[4] @@ -14,9 +13,7 @@ def test_manifest_declares_only_the_bundled_server(self) -> None: self.assertEqual(set(manifest["mcpServers"]), {"profile_and_optimize"}) server = manifest["mcpServers"]["profile_and_optimize"] - self.assertEqual( - server["command"], "${CLAUDE_PLUGIN_ROOT}/server/.venv/bin/python" - ) + self.assertEqual(server["command"], "${CLAUDE_PLUGIN_ROOT}/server/.venv/bin/python") self.assertEqual( server["env"], {"PROFILE_AND_OPTIMIZE_REPO_ROOT": "${CLAUDE_PLUGIN_ROOT}/server"}, diff --git a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_server_smoke.py b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_server_smoke.py index 084d0be..d23bdd3 100644 --- a/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_server_smoke.py +++ b/plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/tests/test_server_smoke.py @@ -141,9 +141,12 @@ def test_runtime_forwards_ack_param_as_cli_flag(self) -> None: "slurm_drain", { "args": [ - "--nodes", "test-node", - "--ns", "test-namespace", - "--bundle", bundle, + "--nodes", + "test-node", + "--ns", + "test-namespace", + "--bundle", + bundle, ], "i_understand_this_substitutes_nodes": True, }, @@ -379,9 +382,12 @@ def test_cluster_mutation_dry_runs_need_no_acknowledgement(self) -> None: ( "perf_tune_report_cell_run", [ - "--campaign", "/tmp/not-used", - "--cell", "cell1", - "--backend", "vllm-sweep", + "--campaign", + "/tmp/not-used", + "--cell", + "cell1", + "--backend", + "vllm-sweep", "--dry-run", ], "--i-understand-this-submits-jobs", @@ -389,8 +395,10 @@ def test_cluster_mutation_dry_runs_need_no_acknowledgement(self) -> None: ( "perf_tune_report_campaign_run", [ - "--config", "/tmp/not-used.yaml", - "--campaign", "/tmp/not-used", + "--config", + "/tmp/not-used.yaml", + "--campaign", + "/tmp/not-used", "--dry-run", ], "--i-understand-this-mutates-cluster", @@ -417,7 +425,9 @@ def test_console_entrypoint_list_matches_mcp_surface(self) -> None: with contextlib.redirect_stdout(buffer): rc = mcp_surface.main(["--json", "list"]) self.assertEqual(rc, 0) - self.assertEqual(set(tool_names()), {tool["name"] for tool in __import__("json").loads(buffer.getvalue())["tools"]}) + self.assertEqual( + set(tool_names()), {tool["name"] for tool in __import__("json").loads(buffer.getvalue())["tools"]} + ) def test_search_tools_return_standard_envelope(self) -> None: result = _search("search_runbooks", ["runbooks", "docs"], "performance", limit=3) @@ -511,9 +521,7 @@ def test_performance_hints_are_exposed_and_searchable(self) -> None: limit=5, ) self.assertEqual(result["returncode"], 0) - self.assertTrue( - any("docs/performance-hints.md" in match for match in result["json"]["matches"]) - ) + self.assertTrue(any("docs/performance-hints.md" in match for match in result["json"]["matches"])) def test_runtime_traps_systemexit_from_argparse_help(self) -> None: """Regression test: argparse `--help` raises SystemExit which would @@ -603,9 +611,7 @@ def collect_leaves(p: _ap.ArgumentParser) -> list[_ap.ArgumentParser]: break self.assertIsNotNone(verb_parser, msg=f"{spec.name}: top-level subparser missing") for leaf in collect_leaves(verb_parser): - option_strings = { - opt for action in leaf._actions for opt in action.option_strings - } + option_strings = {opt for action in leaf._actions for opt in action.option_strings} with self.subTest(name=spec.name, leaf=leaf.prog): self.assertIn( "--json", @@ -635,11 +641,7 @@ def test_read_only_leaf_parsers_expose_no_write_options(self) -> None: def collect_leaves(parser: argparse.ArgumentParser) -> list[argparse.ArgumentParser]: for action in parser._actions: if isinstance(action, argparse._SubParsersAction): - return [ - leaf - for child in action.choices.values() - for leaf in collect_leaves(child) - ] + return [leaf for child in action.choices.values() for leaf in collect_leaves(child)] return [parser] for spec in mcp_surface.derive_tool_specs(): @@ -657,11 +659,7 @@ def collect_leaves(parser: argparse.ArgumentParser) -> list[argparse.ArgumentPar ) self.assertIsNotNone(verb_parser, spec.name) for leaf in collect_leaves(verb_parser): - options = { - option - for action in leaf._actions - for option in action.option_strings - } + options = {option for action in leaf._actions for option in action.option_strings} with self.subTest(name=spec.name, leaf=leaf.prog): self.assertFalse( options & write_options, diff --git a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/cli.py b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/cli.py index 918c73e..a2560b2 100644 --- a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/cli.py +++ b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/cli.py @@ -70,8 +70,7 @@ def build_parser() -> argparse.ArgumentParser: "--stride", type=int, default=None, - help="Bucket modulus (default 32 matches the MOD-32 hang signature). " - "Mutually exclusive with --auto-stride.", + help="Bucket modulus (default 32 matches the MOD-32 hang signature). Mutually exclusive with --auto-stride.", ) stride_group.add_argument( "--auto-stride", @@ -104,14 +103,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--output", type=Path, - help="When set, append the result as one JSONL row to this path. " - "Parent directory must exist.", + help="When set, append the result as one JSONL row to this path. Parent directory must exist.", ) parser.add_argument( "--json", action="store_true", - help="Emit the full result dict to stdout (default just emits a " - "one-line summary).", + help="Emit the full result dict to stdout (default just emits a one-line summary).", ) return parser @@ -158,9 +155,7 @@ def main(argv: list[str] | None = None) -> int: ) return 2 nodelist = [ - line.strip() - for line in args.nodelist_file.read_text(encoding="utf-8").splitlines() - if line.strip() + line.strip() for line in args.nodelist_file.read_text(encoding="utf-8").splitlines() if line.strip() ] # Auto-stride sweep: invoke run_detector once per candidate stride @@ -188,9 +183,7 @@ def main(argv: list[str] | None = None) -> int: per_stride.append(single) aggregated_alerts.extend(single["alerts"]) # Stable sort: largest lag first, then by stride, then by bucket. - aggregated_alerts.sort( - key=lambda a: (-a["lag"], a["stride"], a["lagging_bucket"]) - ) + aggregated_alerts.sort(key=lambda a: (-a["lag"], a["stride"], a["lagging_bucket"])) result = { "schema_version": 1, "mode": "auto_stride", @@ -206,22 +199,16 @@ def main(argv: list[str] | None = None) -> int: else: if aggregated_alerts: buckets_by_stride = { - stride: sorted( - a["lagging_bucket"] for a in aggregated_alerts if a["stride"] == stride - ) + stride: sorted(a["lagging_bucket"] for a in aggregated_alerts if a["stride"] == stride) for stride in candidates } - stride_summary = ", ".join( - f"stride={s}->{buckets_by_stride[s] or '[]'}" for s in candidates - ) + stride_summary = ", ".join(f"stride={s}->{buckets_by_stride[s] or '[]'}" for s in candidates) print( - f"HANG-DETECTED: {len(aggregated_alerts)} alert(s) across " - f"strides {candidates}. {stride_summary}." + f"HANG-DETECTED: {len(aggregated_alerts)} alert(s) across strides {candidates}. {stride_summary}." ) else: print( - f"OK: no stride-pattern lag detected across strides " - f"{candidates} on {result['rank_count']} rank(s)." + f"OK: no stride-pattern lag detected across strides {candidates} on {result['rank_count']} rank(s)." ) return 1 if aggregated_alerts else 0 @@ -248,10 +235,7 @@ def main(argv: list[str] | None = None) -> int: f"Affected buckets: {sorted(a['lagging_bucket'] for a in alerts)}." ) else: - print( - f"OK: no stride-pattern lag detected at stride={stride} " - f"across {result['rank_count']} rank(s)." - ) + print(f"OK: no stride-pattern lag detected at stride={stride} across {result['rank_count']} rank(s).") # Non-zero exit when alerts present so callers can chain. return 1 if result["alerts"] else 0 diff --git a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/detector.py b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/detector.py index aec8684..88fa2bc 100644 --- a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/detector.py +++ b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/detector.py @@ -10,7 +10,7 @@ from __future__ import annotations import json -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -63,13 +63,11 @@ def run_detector( nodelist=nodelist, port=port, ) - alerts = detect_stride_lag( - snapshots, stride=stride, lag_threshold=lag_threshold - ) + alerts = detect_stride_lag(snapshots, stride=stride, lag_threshold=lag_threshold) result: dict[str, Any] = { "schema_version": 1, "jobid": jobid, - "captured_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "captured_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), "stride": stride, "lag_threshold": lag_threshold, "rank_count": len(snapshots), @@ -79,8 +77,7 @@ def run_detector( parent = output_path.parent if not parent.is_dir(): raise FileNotFoundError( - f"output_path parent does not exist: {parent}; " - f"create it explicitly (orchestrator does not auto-mkdir)." + f"output_path parent does not exist: {parent}; create it explicitly (orchestrator does not auto-mkdir)." ) with output_path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(result, sort_keys=True) + "\n") diff --git a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/gpusd_scraper.py b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/gpusd_scraper.py index 92d041e..6c82ed4 100644 --- a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/gpusd_scraper.py +++ b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/gpusd_scraper.py @@ -81,9 +81,7 @@ def scrape_gpusd_snapshot( if live_cluster: return _scrape_live(nodelist or [], port) if fixture_path is None: - raise ValueError( - "scrape_gpusd_snapshot requires fixture_path when live_cluster=False" - ) + raise ValueError("scrape_gpusd_snapshot requires fixture_path when live_cluster=False") if not fixture_path.is_file(): raise FileNotFoundError(f"fixture not found: {fixture_path}") return _scrape_fixture(fixture_path) @@ -140,11 +138,10 @@ def _scrape_live(nodelist: list[str], port: int) -> list[RankSnapshot]: if not nodelist: raise ValueError("live_cluster=True requires a non-empty nodelist") try: - import requests # noqa: F401 (imported here for the live path only) + import requests except ImportError as exc: raise ImportError( - "live_cluster=True requires the `requests` package; " - "install it or use the fixture path" + "live_cluster=True requires the `requests` package; install it or use the fixture path" ) from exc # Group nodes by rack. The cluster's hostname convention is @@ -155,7 +152,7 @@ def _scrape_live(nodelist: list[str], port: int) -> list[RankSnapshot]: racks.setdefault(rack, []).append(host) by_rank: dict[int, RankSnapshot] = {} - for rack, hosts in racks.items(): + for hosts in racks.values(): # One persistent session per rack, reused across hosts. This # caps FDs at len(racks), not len(nodelist). session = requests.Session() @@ -164,15 +161,10 @@ def _scrape_live(nodelist: list[str], port: int) -> list[RankSnapshot]: url = f"http://{host}:{port}/metrics" resp = session.get(url, timeout=5.0) if resp.status_code != 200: - raise RuntimeError( - f"{host}: GET {url} returned {resp.status_code}" - ) + raise RuntimeError(f"{host}: GET {url} returned {resp.status_code}") payload = resp.json() if payload.get("schema_version") != 1: - raise RuntimeError( - f"{host}: unsupported schema_version=" - f"{payload.get('schema_version')}" - ) + raise RuntimeError(f"{host}: unsupported schema_version={payload.get('schema_version')}") for row in payload.get("ranks", []): snap = RankSnapshot( rank=int(row["rank"]), diff --git a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/stride_detector.py b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/stride_detector.py index 4c77ba6..ac6351b 100644 --- a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/stride_detector.py +++ b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/stride_detector.py @@ -109,9 +109,7 @@ def detect_stride_lag( # Need at least two buckets to compare leader vs lagging. return [] - bucket_medians: dict[int, int] = { - b: int(median(s.seq_num for s in group)) for b, group in buckets.items() - } + bucket_medians: dict[int, int] = {b: int(median(s.seq_num for s in group)) for b, group in buckets.items()} leader_bucket = max(bucket_medians, key=lambda b: bucket_medians[b]) leader_median = bucket_medians[leader_bucket] diff --git a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/tests/test_stride_detector.py b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/tests/test_stride_detector.py index d5744dc..cbe2014 100644 --- a/plugins/profile-and-optimize/server/tools/profiling/hang_detector/tests/test_stride_detector.py +++ b/plugins/profile-and-optimize/server/tools/profiling/hang_detector/tests/test_stride_detector.py @@ -21,10 +21,7 @@ class StrideDetectorTest(unittest.TestCase): def test_healthy_cohort_emits_zero_alerts(self) -> None: """All ranks advance synchronously; no bucket lags.""" - snaps = [ - RankSnapshot(rank=r, seq_num=150, timestamp=1.0) - for r in (0, 1, 16, 31, 32, 33, 48, 63, 64, 95, 127) - ] + snaps = [RankSnapshot(rank=r, seq_num=150, timestamp=1.0) for r in (0, 1, 16, 31, 32, 33, 48, 63, 64, 95, 127)] alerts = detect_stride_lag(snaps, stride=32) self.assertEqual(alerts, []) @@ -101,9 +98,7 @@ def test_single_bucket_returns_no_alerts(self) -> None: def test_invalid_stride_raises(self) -> None: with self.assertRaises(ValueError): - detect_stride_lag( - [RankSnapshot(rank=0, seq_num=1, timestamp=1.0)], stride=0 - ) + detect_stride_lag([RankSnapshot(rank=0, seq_num=1, timestamp=1.0)], stride=0) def test_invalid_lag_threshold_raises(self) -> None: with self.assertRaises(ValueError): diff --git a/plugins/profile-and-optimize/server/tools/shared/test_validation_schema.py b/plugins/profile-and-optimize/server/tools/shared/test_validation_schema.py index 77d301c..77d151a 100644 --- a/plugins/profile-and-optimize/server/tools/shared/test_validation_schema.py +++ b/plugins/profile-and-optimize/server/tools/shared/test_validation_schema.py @@ -10,7 +10,7 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from tools.shared import validation_schema # noqa: E402 +from tools.shared import validation_schema def _load_artifact_validation(): diff --git a/plugins/profile-and-optimize/server/tools/slurm/slurm_cli.py b/plugins/profile-and-optimize/server/tools/slurm/slurm_cli.py index de90da8..1ab0698 100644 --- a/plugins/profile-and-optimize/server/tools/slurm/slurm_cli.py +++ b/plugins/profile-and-optimize/server/tools/slurm/slurm_cli.py @@ -670,7 +670,7 @@ def cmd_quiet_window(args: argparse.Namespace) -> int: try: try: drain_proc = RUN(drain_argv, capture=True, timeout=60) - except BaseException as exc: + except BaseException as exc: # noqa: BLE001 - resume must run after interrupts. drain_exception = exc if not isinstance(exc, Exception): pending_exception = exc @@ -720,7 +720,7 @@ def cmd_quiet_window(args: argparse.Namespace) -> int: try: inner_proc = RUN(inner_argv, capture=True, timeout=None) inner_rc = inner_proc.returncode - except BaseException as exc: + except BaseException as exc: # noqa: BLE001 - resume must run after interrupts. inner_exception = exc inner_rc = 1 if not isinstance(exc, Exception): diff --git a/plugins/profile-and-optimize/server/tools/stage-model-parallel.py b/plugins/profile-and-optimize/server/tools/stage-model-parallel.py index 00dfc76..f8b892f 100644 --- a/plugins/profile-and-optimize/server/tools/stage-model-parallel.py +++ b/plugins/profile-and-optimize/server/tools/stage-model-parallel.py @@ -27,14 +27,15 @@ Usage: stage-model-parallel.py s3://perf-lake/saved_experiments/cluster-pvcs/glm51-cache/target /models/glm51 """ + import os -import sys -import time import socket +import sys import threading -from urllib.parse import urlparse +import time from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path +from urllib.parse import urlparse import boto3 from boto3.s3.transfer import TransferConfig @@ -102,13 +103,9 @@ def _download_path(destination_root: Path, prefix: str, object_key: str) -> Path key_prefix = f"{prefix}/" if not object_key.startswith(key_prefix): raise RuntimeError(f"object key is outside requested prefix: {object_key!r}") - relative_key = object_key[len(key_prefix):] + relative_key = object_key[len(key_prefix) :] parts = relative_key.split("/") - if ( - not relative_key - or "\\" in relative_key - or any(part in {"", ".", ".."} for part in parts) - ): + if not relative_key or "\\" in relative_key or any(part in {"", ".", ".."} for part in parts): raise RuntimeError(f"unsafe object key below requested prefix: {object_key!r}") output = destination_root.joinpath(*parts).resolve() if not output.is_relative_to(destination_root): @@ -118,8 +115,11 @@ def _download_path(destination_root: Path, prefix: str, object_key: str) -> Path def make_client(endpoint: str): return boto3.client( - "s3", endpoint_url=endpoint, aws_access_key_id=AK, - aws_secret_access_key=SK, region_name=REGION, + "s3", + endpoint_url=endpoint, + aws_access_key_id=AK, + aws_secret_access_key=SK, + region_name=REGION, config=Config( s3={"addressing_style": "virtual"}, # some providers reject path-style on ListObjectsV2 retries={"max_attempts": 5, "mode": "adaptive"}, @@ -159,16 +159,14 @@ def main(): output = _download_path(destination_root, prefix, key) previous_key = output_keys.get(output) if previous_key is not None: - raise RuntimeError( - f"object keys map to the same destination: {previous_key!r} and {key!r}" - ) + raise RuntimeError(f"object keys map to the same destination: {previous_key!r} and {key!r}") output_keys[output] = key keys.append((key, size, output)) total += size if not keys: raise SystemExit(f"FATAL: no objects under s3://{bucket}/{prefix}/") - print(f"[stage] {len(keys)} objects, {total/1e9:.1f} GB -> {destination_root}", flush=True) + print(f"[stage] {len(keys)} objects, {total / 1e9:.1f} GB -> {destination_root}", flush=True) print(f"[stage] shards={SHARD_CONCURRENCY} multipart={MULTIPART_CONCURRENCY}x{CHUNK_MB}MB", flush=True) t0 = time.time() @@ -189,8 +187,11 @@ def fetch(key, size, out): done["n"] += 1 done["bytes"] += size el = max(time.time() - t0, 1e-6) - print(f"[stage] {done['n']}/{len(keys)} {rel} ({size/1e9:.2f}GB) | " - f"{done['bytes']/1e9:.1f}/{total/1e9:.1f}GB {done['bytes']/1e6/el:.0f} MB/s agg", flush=True) + print( + f"[stage] {done['n']}/{len(keys)} {rel} ({size / 1e9:.2f}GB) | " + f"{done['bytes'] / 1e9:.1f}/{total / 1e9:.1f}GB {done['bytes'] / 1e6 / el:.0f} MB/s agg", + flush=True, + ) with ThreadPoolExecutor(max_workers=SHARD_CONCURRENCY) as ex: futs = [ex.submit(fetch, key, size, output) for key, size, output in keys] @@ -199,8 +200,8 @@ def fetch(key, size, out): el = max(time.time() - t0, 1e-6) print( - f"[stage] DONE {total/1e9:.1f} GB in {el:.0f}s = " - f"{total/1e6/el:.0f} MB/s aggregate -> {destination_root}", + f"[stage] DONE {total / 1e9:.1f} GB in {el:.0f}s = " + f"{total / 1e6 / el:.0f} MB/s aggregate -> {destination_root}", flush=True, ) diff --git a/plugins/profile-and-optimize/server/tools/test_capacity_sizing.py b/plugins/profile-and-optimize/server/tools/test_capacity_sizing.py index 3a85054..7d7b551 100644 --- a/plugins/profile-and-optimize/server/tools/test_capacity_sizing.py +++ b/plugins/profile-and-optimize/server/tools/test_capacity_sizing.py @@ -2,7 +2,6 @@ Cases are anchored on the GB300 MiniMax-M2.7-NVFP4 leaderboard cells and the 3M-TPM thread's own arithmetic (18 pods x 2916 tok/s/pod ~= 50k tok/s = 3M TPM).""" -import math import pytest @@ -15,8 +14,14 @@ def test_thread_18_pods_crosscheck(): # The thread: per-pod 2916 tok/s at c=128 (tok/s/user 22.78), 8-GPU pod, 100% util -> 18 pods. - r = resolve(tpm=TPM_3M, sla_list=[2916 / 128], anchors=[Anchor(128, 2916 / 128)], - gpus_per_pod=8, util=1.0, model="thread-crosscheck") + r = resolve( + tpm=TPM_3M, + sla_list=[2916 / 128], + anchors=[Anchor(128, 2916 / 128)], + gpus_per_pod=8, + util=1.0, + model="thread-crosscheck", + ) row = r.rows[0] assert row.replicas == 18 assert row.gpus == 144 diff --git a/plugins/profile-and-optimize/server/tools/test_hooks.py b/plugins/profile-and-optimize/server/tools/test_hooks.py index b21e130..295fe8b 100644 --- a/plugins/profile-and-optimize/server/tools/test_hooks.py +++ b/plugins/profile-and-optimize/server/tools/test_hooks.py @@ -44,9 +44,7 @@ def _decision(verdict: dict[str, object]) -> str: def _guard(tmp_path: Path, output: str, *, exit_code: int = 0) -> Path: guard = tmp_path / "guard.sh" guard.write_text( - "#!/bin/bash\n" - f"printf '%s\\n' {output!r}\n" - f"exit {exit_code}\n", + f"#!/bin/bash\nprintf '%s\\n' {output!r}\nexit {exit_code}\n", encoding="utf-8", ) return guard diff --git a/plugins/profile-and-optimize/server/tools/test_loader_advisor.py b/plugins/profile-and-optimize/server/tools/test_loader_advisor.py index f4907e9..5f490b6 100644 --- a/plugins/profile-and-optimize/server/tools/test_loader_advisor.py +++ b/plugins/profile-and-optimize/server/tools/test_loader_advisor.py @@ -1,4 +1,5 @@ """Unit tests for the loader auto-select resolver (the single source of truth for loader choice).""" + from tools.loader_advisor import mtp_from_serve_args, render_md, resolve diff --git a/plugins/profile-and-optimize/server/tools/test_runtime_scripts.py b/plugins/profile-and-optimize/server/tools/test_runtime_scripts.py index 216ef38..b776ecf 100644 --- a/plugins/profile-and-optimize/server/tools/test_runtime_scripts.py +++ b/plugins/profile-and-optimize/server/tools/test_runtime_scripts.py @@ -10,20 +10,12 @@ import pytest - REPO_ROOT = Path(__file__).resolve().parents[4] BASH = shutil.which("bash") or "/bin/bash" def _load_stage_model_parallel(): - script = ( - REPO_ROOT - / "plugins" - / "profile-and-optimize" - / "server" - / "tools" - / "stage-model-parallel.py" - ) + script = REPO_ROOT / "plugins" / "profile-and-optimize" / "server" / "tools" / "stage-model-parallel.py" spec = importlib.util.spec_from_file_location("stage_model_parallel", script) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) @@ -181,9 +173,7 @@ def test_nsys_capture_rejects_non_numeric_size_before_arithmetic( } ) - result = subprocess.run( - [BASH, str(script)], env=env, text=True, capture_output=True, check=False - ) + result = subprocess.run([BASH, str(script)], env=env, text=True, capture_output=True, check=False) assert result.returncode == 1 assert "MIN_REP_MB must be a positive integer" in result.stdout @@ -205,10 +195,7 @@ def test_profile_run_dry_run_uses_operator_launcher(tmp_path: Path) -> None: artifact_root = tmp_path / "artifacts" marker = tmp_path / "launcher-ran" launcher = tmp_path / "sentinel-launcher" - launcher.write_text( - "#!/usr/bin/env bash\n" - "touch \"${PROFILE_RUN_SENTINEL:?}\"\n" - ) + launcher.write_text('#!/usr/bin/env bash\ntouch "${PROFILE_RUN_SENTINEL:?}"\n') launcher.chmod(0o755) env = os.environ.copy() env["ART_ROOT"] = str(artifact_root) @@ -240,14 +227,7 @@ def test_profile_run_dry_run_uses_operator_launcher(tmp_path: Path) -> None: assert f"launcher: {launcher} --nodes two\\ words" in result.stdout assert "tools/benchmarks" not in result.stdout + result.stderr assert not marker.exists() - capture_log = ( - artifact_root - / "campaign" - / "llama31_8b" - / "public-dry-run" - / "profiling" - / "capture.log" - ) + capture_log = artifact_root / "campaign" / "llama31_8b" / "public-dry-run" / "profiling" / "capture.log" assert capture_log.is_file() assert f"launcher={launcher} --nodes two\\ words" in capture_log.read_text() diff --git a/plugins/profile-and-optimize/server/tools/test_skill_workflows.py b/plugins/profile-and-optimize/server/tools/test_skill_workflows.py index 4241efb..bed2249 100644 --- a/plugins/profile-and-optimize/server/tools/test_skill_workflows.py +++ b/plugins/profile-and-optimize/server/tools/test_skill_workflows.py @@ -4,7 +4,6 @@ from pathlib import Path - PLUGIN_ROOT = Path(__file__).resolve().parents[2] SKILLS_ROOT = PLUGIN_ROOT / "skills" @@ -27,7 +26,6 @@ def test_optimization_skills_do_not_require_absent_private_helpers() -> None: absent_helpers = { "bench-all-workloads.sh", "bench-with-sol.sh", - "bench-all-workloads.sh", "capture-run-env.sh", "capture-sol-window.sh", "known-good-config-gate.sh", @@ -57,67 +55,18 @@ def test_public_runtime_docs_do_not_claim_absent_capture_helpers() -> None: PLUGIN_ROOT / "server" / "docs" / "learnings" / "evidence-shape.md", PLUGIN_ROOT / "server" / "docs" / "zymtrace-query-hygiene.md", PLUGIN_ROOT / "server" / "tools" / "loader_advisor.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "importers" - / "inference_perf_bench.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "importers" - / "zymtrace_kernels.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "ROOFLINE-METHODOLOGY.md", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "importers" / "inference_perf_bench.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "importers" / "zymtrace_kernels.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "ROOFLINE-METHODOLOGY.md", PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "lake_writer.py", PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "perf_tune_report_cli.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "importers" - / "roofline_sweep.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "importers" - / "variant_ab.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "importers" - / "workloads.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "renderer" - / "champion_select.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "renderer" - / "prefill_decode_roofline.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "renderer" - / "render_report.py", - PLUGIN_ROOT - / "server" - / "tools" - / "perf_tune_report" - / "renderer" - / "render_status.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "importers" / "roofline_sweep.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "importers" / "variant_ab.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "importers" / "workloads.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "renderer" / "champion_select.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "renderer" / "prefill_decode_roofline.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "renderer" / "render_report.py", + PLUGIN_ROOT / "server" / "tools" / "perf_tune_report" / "renderer" / "render_status.py", SKILLS_ROOT / "inference-aa-workload" / "SKILL.md", SKILLS_ROOT / "inference-kernel-profile" / "SKILL.md", SKILLS_ROOT / "inference-kernel-ncu-profile" / "SKILL.md", diff --git a/plugins/profile-and-optimize/server/tools/test_tp_rightsize_advisor.py b/plugins/profile-and-optimize/server/tools/test_tp_rightsize_advisor.py index 06955eb..a3fb61a 100644 --- a/plugins/profile-and-optimize/server/tools/test_tp_rightsize_advisor.py +++ b/plugins/profile-and-optimize/server/tools/test_tp_rightsize_advisor.py @@ -1,6 +1,7 @@ """Unit tests for the TP right-size resolver (single source of truth for the energy/OPEX right-size). Cases are anchored on the FLEET-ENERGY-AUDIT measured fleet (GB300, NVFP4, 2026-06-09).""" + from tools.tp_rightsize_advisor import render_md, resolve diff --git a/plugins/profile-and-optimize/server/tools/tp_rightsize_advisor.py b/plugins/profile-and-optimize/server/tools/tp_rightsize_advisor.py index 2b81e3e..a1153fa 100644 --- a/plugins/profile-and-optimize/server/tools/tp_rightsize_advisor.py +++ b/plugins/profile-and-optimize/server/tools/tp_rightsize_advisor.py @@ -22,6 +22,7 @@ models (they fit a low TP by memory, but prefill compute / KV may still favor higher TP -- verify with a same-node TP A/B before shipping). Pure-Python, no cluster calls (the decision is structural). """ + from __future__ import annotations import argparse @@ -32,9 +33,9 @@ VALID_TP = [1, 2, 4, 8] BYTES_PER_PARAM = {"nvfp4": 0.5, "fp4": 0.5, "mxfp4": 0.5, "fp8": 1.0, "bf16": 2.0, "fp16": 2.0} -TARGET_ACTIVE_PER_GPU_B = 3.0 # >= this = a loaded GPU (from the measured Qwen3-30B TP1 = 3B/GPU @ 820W) -SAFE_ACTIVE_PER_GPU_B = 6.0 # right-size target (loaded but headroom); active/rec_tp ~ this -WEIGHT_FRAC_OF_USABLE = 0.55 # weights must fit in this fraction of usable mem (rest = KV + activations) +TARGET_ACTIVE_PER_GPU_B = 3.0 # >= this = a loaded GPU (from the measured Qwen3-30B TP1 = 3B/GPU @ 820W) +SAFE_ACTIVE_PER_GPU_B = 6.0 # right-size target (loaded but headroom); active/rec_tp ~ this +WEIGHT_FRAC_OF_USABLE = 0.55 # weights must fit in this fraction of usable mem (rest = KV + activations) GPU_HOURLY_USD = 8.6 @@ -78,7 +79,7 @@ def resolve( total_params_b: float, active_params_b: float, dtype: str = "nvfp4", - gpu_mem_gb: float = 284.0, # GB300 + gpu_mem_gb: float = 284.0, # GB300 gmu: float = 0.9, current_tp: int | None = None, ) -> TpResult: @@ -87,14 +88,20 @@ def resolve( min_tp_mem = min_tp_for_memory(weights_gb, gpu_mem_gb, gmu) active_cur = (active_params_b / current_tp) if current_tp else None - over = bool(current_tp and current_tp > min_tp_mem and active_cur is not None and active_cur < TARGET_ACTIVE_PER_GPU_B) + over = bool( + current_tp and current_tp > min_tp_mem and active_cur is not None and active_cur < TARGET_ACTIVE_PER_GPU_B + ) if over: + assert current_tp is not None + assert active_cur is not None # right-size: lowest TP that fits memory + keeps active/GPU loaded (~SAFE target) rec = max(min_tp_mem, _nearest_valid_tp_le(active_params_b / SAFE_ACTIVE_PER_GPU_B)) rec = min(rec, current_tp) # never recommend MORE TP than current when reducing else: - rec = current_tp if current_tp else max(min_tp_mem, _nearest_valid_tp_le(active_params_b / SAFE_ACTIVE_PER_GPU_B)) + rec = ( + current_tp if current_tp else max(min_tp_mem, _nearest_valid_tp_le(active_params_b / SAFE_ACTIVE_PER_GPU_B)) + ) active_rec = active_params_b / rec reduction = (current_tp / rec) if (current_tp and rec) else None @@ -104,34 +111,55 @@ def resolve( confidence = "measured" if (total_params_b <= 40 and active_params_b <= 4) else "extrapolated" gates = [ - Gate("memory_fit", "info", f"{weights_gb:.0f} GB weights ({dtype}) fit at min TP={min_tp_mem} " - f"({gpu_mem_gb:.0f} GB/GPU, gmu={gmu}, weights<= {WEIGHT_FRAC_OF_USABLE:.0%} usable)"), - Gate("active_density", "warn" if over else "pass", - (f"active {active_params_b:.1f}B / current TP{current_tp} = {active_cur:.2f}B/GPU " - f"< {TARGET_ACTIVE_PER_GPU_B:.0f}B target -> GPUs underfilled" if over - else (f"active {active_params_b:.1f}B / TP{current_tp} = {active_cur:.2f}B/GPU (loaded)" if current_tp - else f"active {active_params_b:.1f}B; recommend TP keeps >= {TARGET_ACTIVE_PER_GPU_B:.0f}B/GPU"))), + Gate( + "memory_fit", + "info", + f"{weights_gb:.0f} GB weights ({dtype}) fit at min TP={min_tp_mem} " + f"({gpu_mem_gb:.0f} GB/GPU, gmu={gmu}, weights<= {WEIGHT_FRAC_OF_USABLE:.0%} usable)", + ), + Gate( + "active_density", + "warn" if over else "pass", + ( + f"active {active_params_b:.1f}B / current TP{current_tp} = {active_cur:.2f}B/GPU " + f"< {TARGET_ACTIVE_PER_GPU_B:.0f}B target -> GPUs underfilled" + if over + else ( + f"active {active_params_b:.1f}B / TP{current_tp} = {active_cur:.2f}B/GPU (loaded)" + if current_tp + else f"active {active_params_b:.1f}B; recommend TP keeps >= {TARGET_ACTIVE_PER_GPU_B:.0f}B/GPU" + ) + ), + ), ] reasons: list[str] = [] if over: reasons.append( f"OVER-PROVISIONED: {active_params_b:.1f}B active across TP{current_tp} = {active_cur:.2f}B/GPU " f"(< {TARGET_ACTIVE_PER_GPU_B:.0f}B). The small active-param MoE does not benefit from {current_tp}-way " - f"sharding -> near-idle GPUs + worse per-GPU throughput (measured 2.73x on the 30B-A3B class).") + f"sharding -> near-idle GPUs + worse per-GPU throughput (measured 2.73x on the 30B-A3B class)." + ) reasons.append( f"RIGHT-SIZE to TP{rec} (active {active_rec:.1f}B/GPU): ~{reduction:.1f}x fewer GPUs at matched " - f"throughput" + (f", ~${monthly:,.0f}/standing-replica/month at ${GPU_HOURLY_USD}/GPU-hr." if monthly else ".")) + f"throughput" + + (f", ~${monthly:,.0f}/standing-replica/month at ${GPU_HOURLY_USD}/GPU-hr." if monthly else ".") + ) if confidence == "extrapolated": reasons.append( "CONFIDENCE=extrapolated: total params > ~40B, so the low-TP weights fit but prefill compute / " - "KV may still favor a higher TP -- verify with a same-node TP A/B before shipping.") + "KV may still favor a higher TP -- verify with a same-node TP A/B before shipping." + ) else: if current_tp: - reasons.append(f"TP{current_tp} is right-sized: active {active_cur:.2f}B/GPU >= {TARGET_ACTIVE_PER_GPU_B:.0f}B " - f"(loaded). No change.") + reasons.append( + f"TP{current_tp} is right-sized: active {active_cur:.2f}B/GPU >= {TARGET_ACTIVE_PER_GPU_B:.0f}B " + f"(loaded). No change." + ) else: reasons.append(f"Recommend TP{rec} (active {active_rec:.1f}B/GPU; min TP for memory = {min_tp_mem}).") - return TpResult(rec, over, current_tp, min_tp_mem, active_cur, active_rec, reduction, monthly, confidence, gates, reasons) + return TpResult( + rec, over, current_tp, min_tp_mem, active_cur, active_rec, reduction, monthly, confidence, gates, reasons + ) def render_md(r: TpResult) -> str: @@ -154,7 +182,9 @@ def render_md(r: TpResult) -> str: def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description="Recommend TP + flag over-provisioning (energy/OPEX right-size).") p.add_argument("--total-params-b", type=float, required=True, help="total params in billions") - p.add_argument("--active-params-b", type=float, required=True, help="active params/token in billions (=total for dense)") + p.add_argument( + "--active-params-b", type=float, required=True, help="active params/token in billions (=total for dense)" + ) p.add_argument("--dtype", default="nvfp4", choices=sorted(set(BYTES_PER_PARAM))) p.add_argument("--gpu-mem-gb", type=float, default=284.0, help="per-GPU memory (GB300=284)") p.add_argument("--gmu", type=float, default=0.9) @@ -163,15 +193,22 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--print-tp", action="store_true", help="print only the recommended TP and exit") args = p.parse_args(argv) - r = resolve(total_params_b=args.total_params_b, active_params_b=args.active_params_b, dtype=args.dtype, - gpu_mem_gb=args.gpu_mem_gb, gmu=args.gmu, current_tp=args.current_tp) + r = resolve( + total_params_b=args.total_params_b, + active_params_b=args.active_params_b, + dtype=args.dtype, + gpu_mem_gb=args.gpu_mem_gb, + gmu=args.gmu, + current_tp=args.current_tp, + ) if args.print_tp: print(r.recommended_tp) return 1 if r.over_provisioned else 0 md = render_md(r) sys.stdout.write(md) if args.emit: - d = Path(args.emit); d.mkdir(parents=True, exist_ok=True) + d = Path(args.emit) + d.mkdir(parents=True, exist_ok=True) (d / "tp_rightsize.json").write_text(json.dumps(dataclasses.asdict(r), indent=2) + "\n") (d / "TP-RIGHTSIZE.md").write_text(md) sys.stdout.write(f"\nwrote {d / 'tp_rightsize.json'} + {d / 'TP-RIGHTSIZE.md'}\n") diff --git a/plugins/profile-and-optimize/skills/inference-aa-workload/assets/aa_ttfo_probe.py b/plugins/profile-and-optimize/skills/inference-aa-workload/assets/aa_ttfo_probe.py index 48b6558..a9bc9ca 100644 --- a/plugins/profile-and-optimize/skills/inference-aa-workload/assets/aa_ttfo_probe.py +++ b/plugins/profile-and-optimize/skills/inference-aa-workload/assets/aa_ttfo_probe.py @@ -30,6 +30,7 @@ ``/v1/chat/completions`` path. It requires an explicit unprivileged port and does not follow redirects. Run it inside the serving pod as described above. """ + import argparse import http.client import json @@ -100,8 +101,12 @@ def run_one(url, model, prompt, min_tokens, extra_inputs=None): body = { "model": model, "messages": [{"role": "user", "content": prompt}], - "temperature": 0, "top_p": 1, "stream": True, - "max_tokens": min_tokens + 64, "min_tokens": min_tokens, "ignore_eos": True, + "temperature": 0, + "top_p": 1, + "stream": True, + "max_tokens": min_tokens + 64, + "min_tokens": min_tokens, + "ignore_eos": True, "stream_options": {"include_usage": True}, } if extra_inputs: @@ -171,10 +176,7 @@ def main(): "--url", type=validated_probe_url, default=DEFAULT_URL, - help=( - "loopback HTTP endpoint with an explicit unprivileged port and the exact " - f"{CHAT_COMPLETIONS_PATH} path" - ), + help=(f"loopback HTTP endpoint with an explicit unprivileged port and the exact {CHAT_COMPLETIONS_PATH} path"), ) ap.add_argument("--model", required=True, help="served-model-name") ap.add_argument("--shape", default="aa-1k", choices=list(SHAPES)) @@ -193,17 +195,20 @@ def p50(key): vals = [r[key] for r in fin if r.get(key) is not None] return round(statistics.median(vals), 1) if vals else None - decode = [(r["content_tokens"] + r["reasoning_tokens"]) / r["total_s"] - for r in fin if r["total_s"] > 0] or [0] + decode = [(r["content_tokens"] + r["reasoning_tokens"]) / r["total_s"] for r in fin if r["total_s"] > 0] or [0] + ttft_p50 = p50("ttft_ms") + ttfo_p50 = p50("ttfo_ms") summary = { - "shape": a.shape, "n": len(fin), - "ttft_p50_ms": p50("ttft_ms"), - "ttfo_p50_ms": p50("ttfo_ms"), # None if the model never emitted an answer token (all-reasoning) + "shape": a.shape, + "n": len(fin), + "ttft_p50_ms": ttft_p50, + "ttfo_p50_ms": ttfo_p50, # None if the model never emitted an answer token (all-reasoning) "reasoning_tokens_p50": p50("reasoning_tokens"), "content_tokens_p50": p50("content_tokens"), "decode_tok_s_per_user_mean": round(statistics.mean(decode), 1), - "ttfo_minus_ttft_p50_ms": (round(p50("ttfo_ms") - p50("ttft_ms"), 1) - if p50("ttfo_ms") and p50("ttft_ms") else None), + "ttfo_minus_ttft_p50_ms": ( + round(ttfo_p50 - ttft_p50, 1) if ttfo_p50 is not None and ttft_p50 is not None else None + ), } print("SUMMARY " + json.dumps(summary), flush=True) diff --git a/plugins/profile-and-optimize/skills/inference-aa-workload/assets/repro_artificialanalysis.py b/plugins/profile-and-optimize/skills/inference-aa-workload/assets/repro_artificialanalysis.py index 39e2bb9..7b4fcc5 100644 --- a/plugins/profile-and-optimize/skills/inference-aa-workload/assets/repro_artificialanalysis.py +++ b/plugins/profile-and-optimize/skills/inference-aa-workload/assets/repro_artificialanalysis.py @@ -45,6 +45,11 @@ "aa-100k": (100000, 2000), } AA_SHAPE_ORDER = ("aa-1k", "aa-10k", "aa-100k") +REDACTED_API_KEY = "" +API_KEY_FLAG_ERROR = ( + "custom AIPerf commands must not include --api-key. " + "Use --api-key or set API_KEY or WANDB_INFERENCE_API_KEY instead." +) AA_TEMPERATURE = 0 AA_TOP_P = 1 @@ -142,13 +147,19 @@ def generate_dataset(shape: str, count: int, out_path: Path) -> dict: for _ in range(count): f.write(json.dumps({"text_input": text, "output_length": out_tokens})) f.write("\n") - return {"shape": shape, "rows": count, "path": str(out_path), "used_tiktoken": used_tiktoken} + return { + "shape": shape, + "rows": count, + "path": str(out_path), + "used_tiktoken": used_tiktoken, + } def build_command(cfg: argparse.Namespace, aiperf_cmd: list[str], shape: str) -> list[str]: in_tokens, out_tokens = AA_SHAPES[shape] artifact_dir = str(Path(cfg.artifact_root) / shape) - cmd = list(aiperf_cmd) + [ + cmd = [ + *aiperf_cmd, "profile", "--model", cfg.model, @@ -160,8 +171,6 @@ def build_command(cfg: argparse.Namespace, aiperf_cmd: list[str], shape: str) -> "--url", cfg.url, ] - if cfg.api_key: - cmd += ["--api-key", cfg.api_key] if cfg.tokenizer: cmd += ["--tokenizer", cfg.tokenizer] if cfg.tokenizer_trust_remote_code: @@ -169,7 +178,12 @@ def build_command(cfg: argparse.Namespace, aiperf_cmd: list[str], shape: str) -> if cfg.mode == "dataset-replay": input_file = str(Path(cfg.dataset_dir) / f"{shape}.jsonl") - cmd += ["--input-file", input_file, "--custom-dataset-type", cfg.custom_dataset_type] + cmd += [ + "--input-file", + input_file, + "--custom-dataset-type", + cfg.custom_dataset_type, + ] else: cmd += [ "--synthetic-input-tokens-mean", @@ -182,9 +196,19 @@ def build_command(cfg: argparse.Namespace, aiperf_cmd: list[str], shape: str) -> "0", ] - cmd += ["--extra-inputs", f"temperature:{AA_TEMPERATURE}", "--extra-inputs", f"top_p:{AA_TOP_P}"] + cmd += [ + "--extra-inputs", + f"temperature:{AA_TEMPERATURE}", + "--extra-inputs", + f"top_p:{AA_TOP_P}", + ] if cfg.extra_output_controls: - cmd += ["--extra-inputs", f"min_tokens:{out_tokens}", "--extra-inputs", "ignore_eos:true"] + cmd += [ + "--extra-inputs", + f"min_tokens:{out_tokens}", + "--extra-inputs", + "ignore_eos:true", + ] cmd += [ "--concurrency", str(cfg.concurrency), @@ -196,6 +220,89 @@ def build_command(cfg: argparse.Namespace, aiperf_cmd: list[str], shape: str) -> return cmd +def build_auth_config(cfg: argparse.Namespace, shape: str) -> dict: + """Build a complete AIPerf config whose key comes from the child env.""" + in_tokens, out_tokens = AA_SHAPES[shape] + artifact_dir = str(Path(cfg.artifact_root) / shape) + if cfg.mode == "dataset-replay": + dataset = { + "type": "file", + "path": str(Path(cfg.dataset_dir) / f"{shape}.jsonl"), + "format": cfg.custom_dataset_type, + "entries": cfg.request_count, + "force_min_tokens": True, + } + else: + dataset = { + "type": "synthetic", + "entries": cfg.request_count, + "prompts": { + "isl": {"mean": in_tokens, "stddev": 0}, + "osl": {"mean": out_tokens, "stddev": 0}, + }, + } + extra = {"temperature": AA_TEMPERATURE, "top_p": AA_TOP_P} + if cfg.extra_output_controls: + extra.update({"min_tokens": out_tokens, "ignore_eos": True}) + return { + "schemaVersion": "2.0", + "benchmark": { + "model": cfg.model, + "endpoint": { + "url": cfg.url, + "type": cfg.endpoint_type, + "path": cfg.endpoint, + "api_key": "${OPENAI_API_KEY}", + "streaming": True, + "extra": extra, + }, + "dataset": dataset, + "phases": { + "type": "concurrency", + "concurrency": cfg.concurrency, + "requests": cfg.request_count, + }, + "artifacts": {"dir": artifact_dir}, + }, + } + + +def auth_config_path(cfg: argparse.Namespace, shape: str) -> Path: + """Return the secret-free AIPerf config path without touching disk.""" + return Path(cfg.artifact_root) / shape / "aiperf-auth.json" + + +def write_auth_config(cfg: argparse.Namespace, shape: str) -> Path: + """Write the secret-free config consumed by AIPerf profile.""" + path = auth_config_path(cfg, shape) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(build_auth_config(cfg, shape), indent=2, sort_keys=True) + "\n") + return path + + +def redact_aiperf_command(command: list[str]) -> list[str]: + """Return a copy with every AIPerf API-key value redacted.""" + redacted = list(command) + for index, token in enumerate(redacted): + if token == "--api-key" and index + 1 < len(redacted): + redacted[index + 1] = REDACTED_API_KEY + elif token.startswith("--api-key="): + redacted[index] = f"--api-key={REDACTED_API_KEY}" + return redacted + + +def validate_aiperf_command(command: list[str]) -> None: + """Reject custom commands that could put an API key back in argv.""" + if any("--api-key" in token for token in command): + raise ValueError(API_KEY_FLAG_ERROR) + + +def redact_subprocess_output(text: str | None, api_key: str | None) -> str: + """Remove the managed API key from captured child output.""" + output = text or "" + return output.replace(api_key, REDACTED_API_KEY) if api_key else output + + def build_arg_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--model", default=os.environ.get("MODEL", "moonshotai/Kimi-K2.6")) @@ -272,7 +379,19 @@ def main(argv: list[str] | None = None) -> int: shapes = [s.strip() for s in cfg.shapes.split(",") if s.strip()] for s in shapes: if s not in AA_SHAPES: - print(f"error: unknown shape {s!r}; expected {sorted(AA_SHAPES)}", file=sys.stderr) + print( + f"error: unknown shape {s!r}; expected {sorted(AA_SHAPES)}", + file=sys.stderr, + ) + return 2 + + aiperf_cmd = None + if not cfg.generate_dataset_only: + aiperf_cmd = resolve_aiperf_cmd(cfg.aiperf_bin) + try: + validate_aiperf_command(aiperf_cmd) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) return 2 if cfg.reasoning: @@ -303,17 +422,36 @@ def main(argv: list[str] | None = None) -> int: if cfg.generate_dataset_only: return 0 - aiperf_cmd = resolve_aiperf_cmd(cfg.aiperf_bin) + assert aiperf_cmd is not None validate_auth(cfg.url, cfg.api_key) + if cfg.api_key and ("\n" in cfg.api_key or "\r" in cfg.api_key): + print("error: API keys must not contain newline characters", file=sys.stderr) + return 2 + child_env = os.environ.copy() + if cfg.api_key: + child_env["OPENAI_API_KEY"] = cfg.api_key rc = 0 for s in shapes: cmd = build_command(cfg, aiperf_cmd, s) + if cfg.api_key: + config_path = auth_config_path(cfg, s) + if not cfg.dry_run: + write_auth_config(cfg, s) + cmd += ["--config", str(config_path)] if cfg.dry_run: - print(shlex.join(cmd)) + print(shlex.join(redact_aiperf_command(cmd))) continue print(f"=== running {s} (mode={cfg.mode}, concurrency={cfg.concurrency}) ===") - proc = subprocess.run(cmd, check=False) + proc = subprocess.run( + cmd, + check=False, + env=child_env, + capture_output=True, + text=True, + ) + sys.stdout.write(redact_subprocess_output(proc.stdout, cfg.api_key)) + sys.stderr.write(redact_subprocess_output(proc.stderr, cfg.api_key)) if proc.returncode != 0: print(f"warning: {s} exited {proc.returncode}", file=sys.stderr) rc = proc.returncode diff --git a/plugins/profile-and-optimize/skills/inference-dcgm-correlate/SKILL.md b/plugins/profile-and-optimize/skills/inference-dcgm-correlate/SKILL.md index 3eae99c..391447d 100644 --- a/plugins/profile-and-optimize/skills/inference-dcgm-correlate/SKILL.md +++ b/plugins/profile-and-optimize/skills/inference-dcgm-correlate/SKILL.md @@ -125,10 +125,10 @@ inputs = DcgmCorrelateInputs( cell_dir=cell, sweep_start=start, sweep_end=end, - hw_key="b200_sm100", # GB300: "gb300_nvl72" + hw_key="b200_sm100", # GB300: "gb300_nvl72" pod_label_selector="app=basic-inference", namespace="inference", - expected_n_gpus=8, # GB300 node = 4 (the deploy TP), NOT 8 + expected_n_gpus=8, # GB300 node = 4 (the deploy TP), NOT 8 ) # Dry-run first to print the PromQL the correlator WILL fire: diff --git a/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/clean-driver.sh b/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/clean-driver.sh index a13bd6b..d0520b5 100644 --- a/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/clean-driver.sh +++ b/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/clean-driver.sh @@ -40,7 +40,7 @@ for r in $(seq 1 "${ROUNDS}"); do OUT=$(req) else # c>1: launch CONCURRENCY identical long generations in parallel, wait all - pids=(); outs=() + pids=() for _ in $(seq 1 "${CONCURRENCY}"); do req & pids+=($!); done for p in "${pids[@]}"; do wait "$p" || true; done OUT="(${CONCURRENCY} parallel streams)" diff --git a/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/decode_budget.py b/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/decode_budget.py index e891c21..bedb6ce 100644 --- a/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/decode_budget.py +++ b/plugins/profile-and-optimize/skills/inference-decode-step-budget/assets/decode_budget.py @@ -20,6 +20,7 @@ Stdlib only (sqlite3, json, gzip). Usage: python3 decode_budget.py TRACE [--tpot-ms 6.21] [--tokens-per-step 2] """ + import argparse import collections import gzip @@ -52,30 +53,29 @@ def gap_histogram(merged): bsum = collections.Counter() for g in gaps: gm = g / 1e6 # ns -> ms - k = ("<0.5ms" if gm < 0.5 else "0.5-3ms" if gm < 3 else - "3-8ms" if gm < 8 else "8-16ms" if gm < 16 else ">16ms") + k = "<0.5ms" if gm < 0.5 else "0.5-3ms" if gm < 3 else "3-8ms" if gm < 8 else "8-16ms" if gm < 16 else ">16ms" buckets[k] += 1 bsum[k] += g return gaps, buckets, bsum -def report(label, span_ns, busy_ns, merged, comm_ns=None, parts=None, - tpot_ms=None, tokens_per_step=None): +def report(label, span_ns, busy_ns, merged, comm_ns=None, parts=None, tpot_ms=None, tokens_per_step=None): span_ms = span_ns / 1e6 busy_ms = busy_ns / 1e6 idle_ms = span_ms - busy_ms print(f"\n=== {label} ===") if parts: - ps = " ".join(f"{k}={v/1e6:.0f}ms" for k, v in parts.items()) + ps = " ".join(f"{k}={v / 1e6:.0f}ms" for k, v in parts.items()) print(f"components: {ps}") - print(f"span={span_ms:.0f}ms TRUE-busy={busy_ms:.0f}ms ({100*busy_ms/span_ms:.1f}%) " - f"idle={idle_ms:.0f}ms ({100*idle_ms/span_ms:.1f}%)") + print( + f"span={span_ms:.0f}ms TRUE-busy={busy_ms:.0f}ms ({100 * busy_ms / span_ms:.1f}%) " + f"idle={idle_ms:.0f}ms ({100 * idle_ms / span_ms:.1f}%)" + ) if comm_ns is not None and busy_ns: - print(f"comm (allreduce/allgather) = {comm_ns/1e6:.0f}ms ({100*comm_ns/busy_ns:.1f}% of busy)") + print(f"comm (allreduce/allgather) = {comm_ns / 1e6:.0f}ms ({100 * comm_ns / busy_ns:.1f}% of busy)") gaps, buckets, bsum = gap_histogram(merged) - print("idle-gap buckets (count, total_ms):", - {k: (buckets[k], round(bsum[k] / 1e6, 1)) for k in buckets}) + print("idle-gap buckets (count, total_ms):", {k: (buckets[k], round(bsum[k] / 1e6, 1)) for k in buckets}) # Gate 4: repeating decode-step pattern -> a dominant 3-16ms inter-step bucket stepgaps = [g for g in gaps if 3e6 <= g <= 16e6] @@ -93,18 +93,23 @@ def report(label, span_ns, busy_ns, merged, comm_ns=None, parts=None, per_busy_us = busy_ns / 1e3 / max(n, 1) per_idle_us = sum(stepgaps) / 1e3 / max(n, 1) step_ms = med_gap + per_busy_us / 1e3 - print(f"decode steps~{n} inter-step idle med={med_gap:.2f}ms " - f"per-step: busy~{per_busy_us:.0f}us idle~{per_idle_us:.0f}us cadence~{step_ms:.2f}ms/step") - verdict = ("HOST-BOUND" if per_busy_us / 1e3 < 0.5 * med_gap else - "BALANCED/KERNEL-BOUND") - print(f"classification hint: {verdict} (GPU-busy {per_busy_us:.0f}us vs host-idle {per_idle_us:.0f}us per step)") + print( + f"decode steps~{n} inter-step idle med={med_gap:.2f}ms " + f"per-step: busy~{per_busy_us:.0f}us idle~{per_idle_us:.0f}us cadence~{step_ms:.2f}ms/step" + ) + verdict = "HOST-BOUND" if per_busy_us / 1e3 < 0.5 * med_gap else "BALANCED/KERNEL-BOUND" + print( + f"classification hint: {verdict} (GPU-busy {per_busy_us:.0f}us vs host-idle {per_idle_us:.0f}us per step)" + ) # Gate 3: TPOT reconciliation if tpot_ms: tps = tokens_per_step or 1 modeled = step_ms / tps ok = abs(modeled - tpot_ms) / tpot_ms <= 0.15 - print(f"Gate 3 reconcile: step {step_ms:.2f}ms / {tps} tok = {modeled:.2f} ms/tok " - f"vs driver TPOT {tpot_ms:.2f} ms/tok -> {'OK' if ok else 'MISMATCH (capture suspect)'}") + print( + f"Gate 3 reconcile: step {step_ms:.2f}ms / {tps} tok = {modeled:.2f} ms/tok " + f"vs driver TPOT {tpot_ms:.2f} ms/tok -> {'OK' if ok else 'MISMATCH (capture suspect)'}" + ) if not ok: quality = "RED" notes.append("TPOT reconciliation mismatch >15%") @@ -125,9 +130,11 @@ def tables(): KERNEL = "CUPTI_ACTIVITY_KIND_KERNEL" if KERNEL not in tabs: print("RED: sqlite has no CUPTI_ACTIVITY_KIND_KERNEL (no CUDA-kernel data).") - print(" Gate 0: on GB300 this is usually the CUDA 12.x-image vs 13.x-driver " - "CUPTI skew (CUPTI_ERROR_INVALID_DEVICE), NOT capture hygiene -- grep the kineto/nsys " - "log for 'CUDA versions. CUPTI/Runtime/Driver'; fix = CUDA-13 image or zymtrace.") + print( + " Gate 0: on GB300 this is usually the CUDA 12.x-image vs 13.x-driver " + "CUPTI skew (CUPTI_ERROR_INVALID_DEVICE), NOT capture hygiene -- grep the kineto/nsys " + "log for 'CUDA versions. CUPTI/Runtime/Driver'; fix = CUDA-13 image or zymtrace." + ) return "RED" GRAPH = "CUPTI_ACTIVITY_KIND_GRAPH_TRACE" if "CUPTI_ACTIVITY_KIND_GRAPH_TRACE" in tabs else None MEMCPY = "CUPTI_ACTIVITY_KIND_MEMCPY" if "CUPTI_ACTIVITY_KIND_MEMCPY" in tabs else None @@ -139,34 +146,50 @@ def tables(): SUM(CASE WHEN s.value LIKE '%cudaLaunchKernel%' THEN 1 ELSE 0 END) FROM CUPTI_ACTIVITY_KIND_RUNTIME r JOIN StringIds s ON r.nameId=s.id""") gl, kl = cur.fetchone() - print(f"[graphs] cudaGraphLaunch={gl or 0} cudaLaunchKernel={kl or 0} " - f"-> {'CUDA graphs IN PLAY (GRAPH_TRACE counted below)' if (gl or 0) else 'no graphs'}") + print( + f"[graphs] cudaGraphLaunch={gl or 0} cudaLaunchKernel={kl or 0} " + f"-> {'CUDA graphs IN PLAY (GRAPH_TRACE counted below)' if (gl or 0) else 'no graphs'}" + ) cur.execute(f"SELECT DISTINCT deviceId FROM {KERNEL} ORDER BY deviceId") devs = [r[0] for r in cur.fetchall()] overall = "GREEN" rep_dev = devs[0] if devs else None for dev in devs: - def grab(tbl): + + def grab(tbl, device_id=dev): if not tbl: return [] - cur.execute(f"SELECT start,end FROM {tbl} WHERE deviceId=?", (dev,)) + cur.execute(f"SELECT start,end FROM {tbl} WHERE deviceId=?", (device_id,)) return cur.fetchall() - k = grab(KERNEL); g = grab(GRAPH); mc = grab(MEMCPY) - kb, _ = union_busy(k); gb, _ = union_busy(g) + + k = grab(KERNEL) + g = grab(GRAPH) + mc = grab(MEMCPY) + kb, _ = union_busy(k) + gb, _ = union_busy(g) allb, merged = union_busy(k + g + mc) span = merged[-1][1] - merged[0][0] # comm = allreduce/allgather eager kernels - cur.execute(f"""SELECT SUM(x.e-x.s) FROM (SELECT k.start s,k.end e FROM {KERNEL} k + cur.execute( + f"""SELECT SUM(x.e-x.s) FROM (SELECT k.start s,k.end e FROM {KERNEL} k JOIN StringIds s ON k.shortName=s.id WHERE k.deviceId=? AND (s.value LIKE '%allreduce%' OR s.value LIKE '%ncclDevKernel%' - OR s.value LIKE '%all_reduce%' OR s.value LIKE '%cross_device_reduce%')) x""", (dev,)) + OR s.value LIKE '%all_reduce%' OR s.value LIKE '%cross_device_reduce%')) x""", + (dev,), + ) comm = cur.fetchone()[0] or 0 - only_rep = (dev == rep_dev) - q = report(f"nsys dev{dev}", span, allb, merged, comm_ns=comm, - parts={"KERNEL": kb, "GRAPH": gb}, - tpot_ms=tpot_ms if only_rep else None, - tokens_per_step=tokens_per_step) + only_rep = dev == rep_dev + q = report( + f"nsys dev{dev}", + span, + allb, + merged, + comm_ns=comm, + parts={"KERNEL": kb, "GRAPH": gb}, + tpot_ms=tpot_ms if only_rep else None, + tokens_per_step=tokens_per_step, + ) if q == "RED": overall = "RED" return overall @@ -181,20 +204,33 @@ def analyze_kineto(source, compressed, tpot_ms, tokens_per_step): trace = json.load(source) ev = trace.get("traceEvents", trace if isinstance(trace, list) else []) GPU_CATS = {"kernel", "gpu_memcpy", "gpu_memset"} - gpu = [(int(e["ts"] * 1000), int((e["ts"] + e.get("dur", 0)) * 1000)) # us -> ns - for e in ev if e.get("ph") == "X" and e.get("cat") in GPU_CATS and "ts" in e] + gpu = [ + (int(e["ts"] * 1000), int((e["ts"] + e.get("dur", 0)) * 1000)) # us -> ns + for e in ev + if e.get("ph") == "X" and e.get("cat") in GPU_CATS and "ts" in e + ] if not gpu: print("RED: kineto trace has no GPU kernel/memcpy events (no CUDA-kernel data).") return "RED" graph_launches = sum(1 for e in ev if "cudaGraphLaunch" in (e.get("name") or "")) kern_launches = sum(1 for e in ev if "cudaLaunchKernel" in (e.get("name") or "")) - print(f"[graphs] cudaGraphLaunch={graph_launches} cudaLaunchKernel={kern_launches} " - f"-> {'graphs in play (kineto attributes graph kernels as events)' if graph_launches else 'no graphs'}") - comm = sum(e1 - s1 for s1, e1 in - [(int(e["ts"] * 1000), int((e["ts"] + e.get("dur", 0)) * 1000)) - for e in ev if e.get("ph") == "X" and e.get("cat") in GPU_CATS - and any(x in (e.get("name") or "").lower() - for x in ("allreduce", "nccl", "all_reduce", "reduce_scatter", "all_gather", "allgather"))]) + print( + f"[graphs] cudaGraphLaunch={graph_launches} cudaLaunchKernel={kern_launches} " + f"-> {'graphs in play (kineto attributes graph kernels as events)' if graph_launches else 'no graphs'}" + ) + comm = sum( + e1 - s1 + for s1, e1 in [ + (int(e["ts"] * 1000), int((e["ts"] + e.get("dur", 0)) * 1000)) + for e in ev + if e.get("ph") == "X" + and e.get("cat") in GPU_CATS + and any( + x in (e.get("name") or "").lower() + for x in ("allreduce", "nccl", "all_reduce", "reduce_scatter", "all_gather", "allgather") + ) + ] + ) print("\n[!] KINETO OVERHEAD CAVEAT: the torch profiler slows decode (~6x on") print(" TP=8 NVFP4+graph deploys) and inflates kernel durations, so the") print(" ABSOLUTE GPU-busy% below is NOT trustworthy (reads high vs reality).") @@ -203,17 +239,26 @@ def analyze_kineto(source, compressed, tpot_ms, tokens_per_step): print(" backend (--profiler-config.profiler=cuda + nsys). See the skill.") busy, merged = union_busy(gpu) span = merged[-1][1] - merged[0][0] - return report("kineto (GPU stream union; busy% overhead-distorted)", span, busy, - merged, comm_ns=comm, tpot_ms=tpot_ms, tokens_per_step=tokens_per_step) + return report( + "kineto (GPU stream union; busy% overhead-distorted)", + span, + busy, + merged, + comm_ns=comm, + tpot_ms=tpot_ms, + tokens_per_step=tokens_per_step, + ) def main(): ap = argparse.ArgumentParser() ap.add_argument("trace", help="nsys .sqlite or kineto .json[.gz]") - ap.add_argument("--tpot-ms", type=float, default=None, - help="driver-measured TPOT (ms/tok) for the reconciliation gate") - ap.add_argument("--tokens-per-step", type=float, default=1, - help="tokens emitted per decode step (MTP accept_len; ~2 for K=1)") + ap.add_argument( + "--tpot-ms", type=float, default=None, help="driver-measured TPOT (ms/tok) for the reconciliation gate" + ) + ap.add_argument( + "--tokens-per-step", type=float, default=1, help="tokens emitted per decode step (MTP accept_len; ~2 for K=1)" + ) a = ap.parse_args() p = a.trace try: @@ -226,8 +271,9 @@ def main(): elif p.endswith(".json") or p.endswith(".json.gz"): q = analyze_kineto(source, p.endswith(".gz"), a.tpot_ms, a.tokens_per_step) else: - sys.exit("unknown trace type (want .sqlite or .json[.gz]). For nsys: " - "`nsys export --type sqlite ` first.") + sys.exit( + "unknown trace type (want .sqlite or .json[.gz]). For nsys: `nsys export --type sqlite ` first." + ) finally: stdin_buffer = getattr(sys.stdin, "buffer", sys.stdin) if source is not stdin_buffer: diff --git a/plugins/profile-and-optimize/skills/inference-graph-diff/SKILL.md b/plugins/profile-and-optimize/skills/inference-graph-diff/SKILL.md index a69877d..b54287d 100644 --- a/plugins/profile-and-optimize/skills/inference-graph-diff/SKILL.md +++ b/plugins/profile-and-optimize/skills/inference-graph-diff/SKILL.md @@ -82,11 +82,14 @@ Repeat with the alternate config to get `side-B.log`. ```python import re, pathlib + + def extract_fx_graphs(log_path): text = pathlib.Path(log_path).read_text(errors="replace") graphs = re.findall(r"=== FX GRAPH ===\n(.*?)\n=== END FX GRAPH ===", text, re.DOTALL) return graphs + a_graphs = extract_fx_graphs("side-A.log") b_graphs = extract_fx_graphs("side-B.log") ``` diff --git a/plugins/profile-and-optimize/skills/inference-workload-profile/tools/profile-to-corpus.py b/plugins/profile-and-optimize/skills/inference-workload-profile/tools/profile-to-corpus.py index 992a3d1..52c2282 100644 --- a/plugins/profile-and-optimize/skills/inference-workload-profile/tools/profile-to-corpus.py +++ b/plugins/profile-and-optimize/skills/inference-workload-profile/tools/profile-to-corpus.py @@ -25,6 +25,7 @@ ./profile-to-corpus.py --profile workload-profile.json --out-plan corpus-plan.sh [--total 54000] ./profile-to-corpus.py --profile workload-profile.json --traffic redacted.jsonl --out corpus.jsonl """ + import argparse import json import shlex @@ -35,7 +36,7 @@ CLASS_TO_DATASETS = { "chat": ["ultrachat", "sharegpt"], "code": ["opencodeinstruct", "codealpaca-20k"], - "structured-rewrite": ["opc"], # opc = code-edit/diff-flavored + "structured-rewrite": ["opc"], # opc = code-edit/diff-flavored } @@ -49,16 +50,14 @@ def close_cli_text(stream) -> None: stream.close() -def emit_weighted_plan(profile: dict, total: int, out_plan: str, specforge_dir: str, - out_data: str): +def emit_weighted_plan(profile: dict, total: int, out_plan: str, specforge_dir: str, out_data: str): mix = profile.get("content_class_mix", {}) grand = sum(mix.values()) or 1 quoted_specforge_dir = shlex.quote(specforge_dir) quoted_out_data = shlex.quote(out_data) lines = [ "#!/usr/bin/env bash", - "# corpus-plan.sh -- WS-C2 weighted-dataset plan (auto-generated from a " - "workload profile).", + "# corpus-plan.sh -- WS-C2 weighted-dataset plan (auto-generated from a workload profile).", "# Run inside the SpecForge env (e.g. via b1-prepare-data.sbatch's container).", "# Produces a profile-matched conversations.jsonl to use as run-offline.sh DATA_PATH.", "set -euo pipefail", @@ -72,23 +71,21 @@ def emit_weighted_plan(profile: dict, total: int, out_plan: str, specforge_dir: ] for cls, count in sorted(mix.items()): frac = count / grand - cls_total = int(round(frac * total)) + cls_total = round(frac * total) datasets = CLASS_TO_DATASETS.get(cls, ["ultrachat"]) per = max(1, cls_total // len(datasets)) - status = (f"[corpus] class={cls} frac={frac:.2f} n~={cls_total} " - f"via {','.join(datasets)}") + status = f"[corpus] class={cls} frac={frac:.2f} n~={cls_total} via {','.join(datasets)}" lines.append(f"printf '%s\\n' {shlex.quote(status)}") for ds in datasets: lines.append( - f'python scripts/prepare_data.py --dataset {shlex.quote(ds)} ' + f"python scripts/prepare_data.py --dataset {shlex.quote(ds)} " f'--sample-size {per} --output-path "$OUT" || true' ) # prepare_data.py writes _train.jsonl; collect them lines.append(f'PARTS+=("$OUT/{ds}_train.jsonl")') lines += [ f'cat "${{PARTS[@]}}" > {quoted_out_data}', - f"printf '%s lines=%s\\n' {shlex.quote(f'[corpus] wrote {out_data}')} " - f'"$(wc -l < {quoted_out_data})"', + f"printf '%s lines=%s\\n' {shlex.quote(f'[corpus] wrote {out_data}')} \"$(wc -l < {quoted_out_data})\"", ] output = open_cli_text(out_plan, "w") try: @@ -98,7 +95,7 @@ def emit_weighted_plan(profile: dict, total: int, out_plan: str, specforge_dir: print(f"== wrote weighted-dataset plan {out_plan} ==") print("Profile-matched mix:") for cls, count in sorted(mix.items()): - print(f" {cls:<20} {count/grand:6.1%} -> {CLASS_TO_DATASETS.get(cls,['ultrachat'])}") + print(f" {cls:<20} {count / grand:6.1%} -> {CLASS_TO_DATASETS.get(cls, ['ultrachat'])}") print(f"\nRun it in the SpecForge container (cluster). DATA_PATH = {out_data}") @@ -118,7 +115,7 @@ def convert_traffic(profile: dict, traffic: str, out_data: str): continue try: rec = json.loads(line) - except Exception: + except json.JSONDecodeError: skipped += 1 continue convs = [] @@ -130,13 +127,12 @@ def convert_traffic(profile: dict, traffic: str, out_data: str): comp = rec.get("completion") or rec.get("response") if isinstance(comp, str) and comp: convs.append({"role": "assistant", "content": comp}) - elif "conversations" in rec: # already in schema + elif "conversations" in rec: # already in schema convs = rec["conversations"] - if len(convs) < 2: # need at least user+assistant to supervise + if len(convs) < 2: # need at least user+assistant to supervise skipped += 1 continue - fout.write(json.dumps({"id": rec.get("id", f"traffic-{i}"), - "conversations": convs}) + "\n") + fout.write(json.dumps({"id": rec.get("id", f"traffic-{i}"), "conversations": convs}) + "\n") n += 1 finally: close_cli_text(fin) @@ -144,18 +140,23 @@ def convert_traffic(profile: dict, traffic: str, out_data: str): print(f"== wrote {out_data}: {n} conversations ({skipped} skipped) ==") if n == 0: sys.exit(2) - print("This is a true profile-matched corpus (real traffic). Use as run-offline.sh " - "DATA_PATH. NOTE: ensure the traffic was redacted upstream (PII / secrets).") + print( + "This is a true profile-matched corpus (real traffic). Use as run-offline.sh " + "DATA_PATH. NOTE: ensure the traffic was redacted upstream (PII / secrets)." + ) def main(): ap = argparse.ArgumentParser() ap.add_argument("--profile", required=True) ap.add_argument("--traffic", default=None, help="redacted traffic jsonl (mode b)") - ap.add_argument("--out", dest="out_data", default="corpus.jsonl", - help="output conversations jsonl (mode b) or final concat path (mode a)") - ap.add_argument("--out-plan", default="corpus-plan.sh", - help="weighted-dataset plan script (mode a)") + ap.add_argument( + "--out", + dest="out_data", + default="corpus.jsonl", + help="output conversations jsonl (mode b) or final concat path (mode a)", + ) + ap.add_argument("--out-plan", default="corpus-plan.sh", help="weighted-dataset plan script (mode a)") ap.add_argument("--total", type=int, default=54000, help="target corpus size (mode a)") ap.add_argument("--specforge-dir", default="/mnt/data/eagle3-train/SpecForge") args = ap.parse_args() @@ -169,8 +170,7 @@ def main(): if args.traffic: convert_traffic(profile, args.traffic, args.out_data) else: - emit_weighted_plan(profile, args.total, args.out_plan, args.specforge_dir, - args.out_data) + emit_weighted_plan(profile, args.total, args.out_plan, args.specforge_dir, args.out_data) if __name__ == "__main__": diff --git a/plugins/profile-and-optimize/skills/inference-workload-profile/tools/workload-profile.py b/plugins/profile-and-optimize/skills/inference-workload-profile/tools/workload-profile.py index e4283e3..ae890be 100644 --- a/plugins/profile-and-optimize/skills/inference-workload-profile/tools/workload-profile.py +++ b/plugins/profile-and-optimize/skills/inference-workload-profile/tools/workload-profile.py @@ -23,7 +23,9 @@ Usage: ./workload-profile.py --in access.jsonl --out workload-profile.json [--tokenizer ] [--keep-samples 0] """ + import argparse +import importlib import json import math import re @@ -53,12 +55,18 @@ def classify(text: str) -> str: """Heuristic content class so the corpus builder can match the mix.""" if not text: return "chat" - code_markers = ("```", "def ", "function ", "class ", "import ", "#include", - "{", "};", "") + code_markers = ("```", "def ", "function ", "class ", "import ", "#include", "{", "};", "") n_code = sum(text.count(m) for m in code_markers) # structured-rewrite: a request that echoes a large block it wants lightly edited - rewrite_markers = ("rewrite", "edit", "change every", "replace all", "apply this diff", - "modify the following", "refactor") + rewrite_markers = ( + "rewrite", + "edit", + "change every", + "replace all", + "apply this diff", + "modify the following", + "refactor", + ) is_rewrite = any(m in text.lower() for m in rewrite_markers) and len(text) > 400 if is_rewrite: return "structured-rewrite" @@ -73,7 +81,7 @@ def percentiles(xs, ps=(50, 90, 99)): s = sorted(xs) out = {} for p in ps: - k = min(len(s) - 1, int(round((p / 100.0) * (len(s) - 1)))) + k = min(len(s) - 1, round((p / 100.0) * (len(s) - 1))) out[f"p{p}"] = s[k] return out @@ -92,16 +100,22 @@ def recommend_method(class_mix: dict, out_p50: int | None) -> dict: code_frac = class_mix.get("code", 0) / total if rewrite_frac >= 0.30: method = "predicted_outputs" - why = (f"{rewrite_frac:.0%} structured-rewrite traffic -> output is largely " - "known a priori; Predicted Outputs (WS-B) gives the biggest win.") + why = ( + f"{rewrite_frac:.0%} structured-rewrite traffic -> output is largely " + "known a priori; Predicted Outputs (WS-B) gives the biggest win." + ) elif code_frac >= 0.40: method = "eagle3" - why = (f"{code_frac:.0%} code traffic -> high local predictability; a draft head " - "trained on the code-weighted corpus (EAGLE3) should accept well.") + why = ( + f"{code_frac:.0%} code traffic -> high local predictability; a draft head " + "trained on the code-weighted corpus (EAGLE3) should accept well." + ) else: method = "eagle3" - why = ("general chat-dominant -> EAGLE3 draft head on the profile-matched corpus; " - "fall back to MTP if the model ships a built-in head.") + why = ( + "general chat-dominant -> EAGLE3 draft head on the profile-matched corpus; " + "fall back to MTP if the model ships a built-in head." + ) return {"method": method, "rationale": why} @@ -139,8 +153,8 @@ def build_aa_profile() -> dict: "recommended_spec_decode": { "method": "eagle3", "rationale": "AA long-context chat/reasoning shapes -> EAGLE3 draft head on " - "an AA-length-weighted corpus; fall back to the model's built-in " - "MTP if a trained head loses the acceptance A/B.", + "an AA-length-weighted corpus; fall back to the model's built-in " + "MTP if a trained head loses the acceptance A/B.", }, "redacted_samples": {}, } @@ -148,16 +162,17 @@ def build_aa_profile() -> dict: def main(): ap = argparse.ArgumentParser() - ap.add_argument("--in", dest="inp", default=None, - help="OpenAI-style access JSONL (omit when using --aa-shapes)") - ap.add_argument("--aa-shapes", action="store_true", - help="synthesize the profile from the Artificial Analysis 1k/10k/100k " - "shapes instead of an access log") + ap.add_argument("--in", dest="inp", default=None, help="OpenAI-style access JSONL (omit when using --aa-shapes)") + ap.add_argument( + "--aa-shapes", + action="store_true", + help="synthesize the profile from the Artificial Analysis 1k/10k/100k shapes instead of an access log", + ) ap.add_argument("--out", default="workload-profile.json") - ap.add_argument("--tokenizer", default=None, - help="HF tokenizer id for exact token counts (else ~4 chars/token)") - ap.add_argument("--keep-samples", type=int, default=0, - help="retain N redacted exemplars per class for the corpus builder") + ap.add_argument("--tokenizer", default=None, help="HF tokenizer id for exact token counts (else ~4 chars/token)") + ap.add_argument( + "--keep-samples", type=int, default=0, help="retain N redacted exemplars per class for the corpus builder" + ) args = ap.parse_args() if args.aa_shapes: @@ -168,9 +183,8 @@ def main(): finally: close_cli_text(output) print(f"== wrote {args.out} (profile_source=aa-shapes) ==") - print(json.dumps({k: profile[k] for k in - ("bench_shapes", "recommended_spec_decode")}, indent=2)) - print("\nNext: profile-to-corpus.py --profile %s -> AA-length-weighted corpus." % args.out) + print(json.dumps({k: profile[k] for k in ("bench_shapes", "recommended_spec_decode")}, indent=2)) + print(f"\nNext: profile-to-corpus.py --profile {args.out} -> AA-length-weighted corpus.") return if not args.inp: @@ -180,10 +194,10 @@ def main(): tok = None if args.tokenizer: try: - from transformers import AutoTokenizer - tok = AutoTokenizer.from_pretrained(args.tokenizer) - except Exception as e: - print(f"[warn] tokenizer load failed ({e}); using char estimate", file=sys.stderr) + auto_tokenizer = importlib.import_module("transformers").AutoTokenizer + tok = auto_tokenizer.from_pretrained(args.tokenizer) + except Exception as exc: # noqa: BLE001 - optional tokenizer failures use the documented fallback. + print(f"[warn] tokenizer load failed ({exc}); using char estimate", file=sys.stderr) def count(text): if tok is not None and text: @@ -204,7 +218,7 @@ def count(text): continue try: rec = json.loads(line) - except Exception: + except json.JSONDecodeError: skipped += 1 continue # shape (b): pre-counted @@ -214,8 +228,7 @@ def count(text): cls = rec.get("content_class") or "chat" elif "messages" in rec: in_text = "\n".join( - m.get("content", "") if isinstance(m.get("content"), str) else "" - for m in rec["messages"] + m.get("content", "") if isinstance(m.get("content"), str) else "" for m in rec["messages"] ) out_text = rec.get("completion") or rec.get("response") or "" il, ol = count(in_text), count(out_text) @@ -237,8 +250,10 @@ def count(text): close_cli_text(source) if n == 0: - print("FATAL: no usable records (need `messages`/`completion` or " - "`prompt_tokens`/`completion_tokens`)", file=sys.stderr) + print( + "FATAL: no usable records (need `messages`/`completion` or `prompt_tokens`/`completion_tokens`)", + file=sys.stderr, + ) sys.exit(2) out_p50 = percentiles(out_lens)["p50"] @@ -252,8 +267,10 @@ def count(text): "isl_buckets": dict(isl_buckets), "osl_buckets": dict(osl_buckets), "bench_shapes": sorted( - {(percentiles(in_lens)["p50"], percentiles(out_lens)["p50"]), - (percentiles(in_lens)["p90"], percentiles(out_lens)["p90"])} + { + (percentiles(in_lens)["p50"], percentiles(out_lens)["p50"]), + (percentiles(in_lens)["p90"], percentiles(out_lens)["p90"]), + } ), "recommended_spec_decode": recommend_method(class_mix, out_p50), "redacted_samples": samples if args.keep_samples else {}, @@ -264,10 +281,8 @@ def count(text): finally: close_cli_text(output) print(f"== wrote {args.out} ==") - print(json.dumps({k: profile[k] for k in - ("n_requests", "content_class_mix", "recommended_spec_decode")}, - indent=2)) - print("\nNext: profile-to-corpus.py --profile %s -> hit-rate-matched corpus." % args.out) + print(json.dumps({k: profile[k] for k in ("n_requests", "content_class_mix", "recommended_spec_decode")}, indent=2)) + print(f"\nNext: profile-to-corpus.py --profile {args.out} -> hit-rate-matched corpus.") if __name__ == "__main__": diff --git a/plugins/profile-and-optimize/skills/mirage-graph-coverage/assets/mk-graph-coverage-audit.py b/plugins/profile-and-optimize/skills/mirage-graph-coverage/assets/mk-graph-coverage-audit.py index 2eeb177..e4b816f 100644 --- a/plugins/profile-and-optimize/skills/mirage-graph-coverage/assets/mk-graph-coverage-audit.py +++ b/plugins/profile-and-optimize/skills/mirage-graph-coverage/assets/mk-graph-coverage-audit.py @@ -19,6 +19,7 @@ GENDIR default: /work/mirage-perop2/demo/deepseek_v3 Exit: 0 = PASS (every critical tensor consumed as input), 1 = FAIL, 2 = no artifacts. """ + import argparse import datetime import glob @@ -60,7 +61,7 @@ def parse_consumed(task_graph_json): for t in d.get("all_tasks", []): tt = t.get("task_type") for io, refs, tts in (("inputs", in_refs, in_tt), ("outputs", out_refs, out_tt)): - for e in (t.get(io) or []): + for e in t.get(io) or []: bp = e.get("base_ptr") if isinstance(e, dict) else None if bp: refs[bp] += 1 @@ -76,15 +77,27 @@ def audit_rank(gendir, rank, critical): if not (os.path.exists(kcu) and os.path.exists(tgj)): return None declared = parse_declared(kcu) - in_refs, out_refs, in_tt, out_tt, ntasks = parse_consumed(tgj) + in_refs, out_refs, in_tt, _out_tt, ntasks = parse_consumed(tgj) dead = sorted(t for t in declared if in_refs.get(t, 0) == 0 and out_refs.get(t, 0) == 0) write_only = sorted(t for t in declared if out_refs.get(t, 0) > 0 and in_refs.get(t, 0) == 0) - crit = {c: {"declared": c in declared, "in_refs": in_refs.get(c, 0), - "consumers": sorted(x for x in in_tt.get(c, set()) if x is not None), - "out_refs": out_refs.get(c, 0)} for c in critical} - return {"rank": rank, "ntasks": ntasks, "n_declared": len(declared), - "dead": dead, "write_only": write_only, "crit": crit, - "mtime": os.path.getmtime(tgj)} + crit = { + c: { + "declared": c in declared, + "in_refs": in_refs.get(c, 0), + "consumers": sorted(x for x in in_tt.get(c, set()) if x is not None), + "out_refs": out_refs.get(c, 0), + } + for c in critical + } + return { + "rank": rank, + "ntasks": ntasks, + "n_declared": len(declared), + "dead": dead, + "write_only": write_only, + "crit": crit, + "mtime": os.path.getmtime(tgj), + } def main(): @@ -97,8 +110,12 @@ def main(): if a.ranks: ranks = [int(x) for x in a.ranks.split(",") if x.strip()] else: - ranks = sorted(int(re.search(r"task_graph_(\d+)\.json", p).group(1)) - for p in glob.glob(os.path.join(a.gendir, "task_graph_*.json"))) + ranks = [] + for path in glob.glob(os.path.join(a.gendir, "task_graph_*.json")): + match = re.search(r"task_graph_(\d+)\.json", path) + if match is not None: + ranks.append(int(match.group(1))) + ranks.sort() if not ranks: print(f"NO ARTIFACTS in {a.gendir}") sys.exit(2) @@ -111,7 +128,7 @@ def main(): print(f"\n## rank {r}: MISSING artifacts") fail = True continue - mt = datetime.datetime.fromtimestamp(res["mtime"], datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + mt = datetime.datetime.fromtimestamp(res["mtime"], datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ") print(f"\n## rank {r} tasks={res['ntasks']} declared_tensors={res['n_declared']} task_graph_mtime={mt}") for c in critical: ci = res["crit"][c] @@ -122,13 +139,17 @@ def main(): status = "OK" else: status = "not-declared" - print(f" [{status}] {c}: declared={ci['declared']} input_refs={ci['in_refs']} " - f"consumer_task_types={ci['consumers']} output_refs={ci['out_refs']}") + print( + f" [{status}] {c}: declared={ci['declared']} input_refs={ci['in_refs']} " + f"consumer_task_types={ci['consumers']} output_refs={ci['out_refs']}" + ) if res["write_only"]: print(f" INFO write-only (produced, 0 input refs) [{len(res['write_only'])}]: {res['write_only'][:20]}") if res["dead"]: print(f" INFO dead (declared, 0 refs) [{len(res['dead'])}]: {res['dead'][:20]}") - print(f"\n# VERDICT: {'FAIL - a critical tensor is declared but never consumed as input (missing wiring)' if fail else 'PASS - every critical tensor is consumed as input by >=1 task'}") + print( + f"\n# VERDICT: {'FAIL - a critical tensor is declared but never consumed as input (missing wiring)' if fail else 'PASS - every critical tensor is consumed as input by >=1 task'}" + ) sys.exit(1 if fail else 0) diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..bfed1a8 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,28 @@ +{ + "include": [ + "plugins/profile-and-optimize/server", + "plugins/profile-and-optimize/hooks", + "plugins/profile-and-optimize/skills", + "scripts" + ], + "exclude": [ + "**/.venv/**", + "**/__pycache__/**", + "**/.pytest_cache/**", + "**/build/**", + "**/dist/**", + "**/*.egg-info/**", + "**/experiments/**", + "**/test_*.py", + "**/tests/**", + "**/testdata/**" + ], + "extraPaths": [ + "plugins/profile-and-optimize/server", + "plugins/profile-and-optimize/server/tools/profile_and_optimize_mcp/src" + ], + "pythonVersion": "3.11", + "venvPath": "plugins/profile-and-optimize/server", + "venv": ".venv", + "typeCheckingMode": "basic" +} diff --git a/requirements/check.txt b/requirements/check.txt index 25947ea..18d3646 100644 --- a/requirements/check.txt +++ b/requirements/check.txt @@ -2,5 +2,7 @@ PyYAML==6.0.3 click==8.4.2 jsonschema==4.26.0 +ruff==0.16.3 +shellcheck-py==0.11.0.1 strictyaml==1.7.3 skills-ref @ git+https://github.com/agentskills/agentskills.git@69ef37e9424c0a7ea9dd2293b559e43ec8176379#subdirectory=skills-ref diff --git a/scripts/check-ci-status.py b/scripts/check-ci-status.py index 1feba66..8e3843b 100644 --- a/scripts/check-ci-status.py +++ b/scripts/check-ci-status.py @@ -27,14 +27,8 @@ def main() -> int: and run.get("headBranch") == "main" and run.get("headSha") == sha ] - if not any( - run.get("status") == "completed" and run.get("conclusion") == "success" - for run in matching - ): - states = ", ".join( - f"{run.get('status')}/{run.get('conclusion')}" - for run in matching - ) or "missing" + if not any(run.get("status") == "completed" and run.get("conclusion") == "success" for run in matching): + states = ", ".join(f"{run.get('status')}/{run.get('conclusion')}" for run in matching) or "missing" print( f"release: no successful ci push workflow run for {sha} ({states})", file=sys.stderr, diff --git a/scripts/check-shell.sh b/scripts/check-shell.sh new file mode 100755 index 0000000..ad37911 --- /dev/null +++ b/scripts/check-shell.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Run ShellCheck with the contributor venv when available. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +VENV_SHELLCHECK="${REPO_ROOT}/plugins/profile-and-optimize/server/.venv/bin/shellcheck" + +if [[ -x "${VENV_SHELLCHECK}" ]]; then + checker="${VENV_SHELLCHECK}" +elif command -v shellcheck >/dev/null 2>&1; then + checker="$(command -v shellcheck)" +else + printf '%s\n' \ + '[FAIL] ShellCheck is not installed. Run: bash plugins/profile-and-optimize/server/install.sh --with-dev' >&2 + exit 2 +fi + +if (( $# > 0 )); then + exec "${checker}" "$@" +fi + +files=() +while IFS= read -r -d '' path; do + files+=("${path}") +done < <(git -C "${REPO_ROOT}" ls-files -z '*.sh') + +if (( ${#files[@]} == 0 )); then + exit 0 +fi + +cd "${REPO_ROOT}" +exec "${checker}" "${files[@]}" diff --git a/scripts/check-version-transition.py b/scripts/check-version-transition.py index 0543cd5..867f3ae 100644 --- a/scripts/check-version-transition.py +++ b/scripts/check-version-transition.py @@ -10,10 +10,7 @@ import sys from pathlib import Path - -VERSION_PATTERN = re.compile( - r"^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$" -) +VERSION_PATTERN = re.compile(r"^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$") LEGACY_VERSION_PATH = "plugins/profile-and-optimize/.claude-plugin/plugin.json" @@ -22,8 +19,7 @@ def git(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]: ["git", *args], check=check, text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, ) @@ -45,9 +41,7 @@ def parent_of(ref: str) -> str | None: return result.stdout.strip() shallow = git("rev-parse", "--is-shallow-repository", check=False) if shallow.returncode == 0 and shallow.stdout.strip() == "true": - raise ValueError( - f"cannot resolve the parent of {ref} in a shallow repository. Fetch full history first" - ) + raise ValueError(f"cannot resolve the parent of {ref} in a shallow repository. Fetch full history first") return None diff --git a/scripts/check_docs.py b/scripts/check_docs.py index 4052b94..9a19534 100644 --- a/scripts/check_docs.py +++ b/scripts/check_docs.py @@ -34,6 +34,7 @@ python3 scripts/check_docs.py """ + from __future__ import annotations import fnmatch @@ -57,15 +58,27 @@ ) STALE_PHRASES = ("once this repo has a remote", "coming soon") MARKER_RE = re.compile(r"\b(TODO|FIXME|TBD)\b") -DASH_RE = re.compile("[–—]") # en dash (U+2013), em dash (U+2014) +DASH_RE = re.compile(r"[\u2013\u2014]") # U+2013 and U+2014 VALUE_BAR = "adversarially-confirmed to add value" VALUE_BAR_DOCS = ("AGENTS.md", "CLAUDE.md", "README.md") -SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", - ".pytest_cache", ".mypy_cache", "dist", "build", "site-packages"} +SKIP_DIRS = { + ".git", + ".venv", + "venv", + "node_modules", + "__pycache__", + ".pytest_cache", + ".mypy_cache", + "dist", + "build", + "site-packages", +} # Spans where a semicolon is legitimate (code, inline code, links, entities). # Blanked before the prose-semicolon scan, newlines kept so line numbers hold. -CODE_SPAN = re.compile(r"```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`|" - r"\]\([^)]*\)|https?://[^\s)]+|&[#a-zA-Z0-9]+;") +CODE_SPAN = re.compile( + r"```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`|" + r"\]\([^)]*\)|https?://[^\s)]+|&[#a-zA-Z0-9]+;" +) def _line(text: str, idx: int) -> int: @@ -92,8 +105,7 @@ def check_doc(path: Path, cfg: dict) -> list[str]: for old in cfg.get("old_names", []): for m in re.finditer(re.escape(old) + r"\b", text): - out.append(f"{rel}:{_line(text, m.start())}: stale repo name " - f"'{old}' (renamed to '{cfg['repo']}')") + out.append(f"{rel}:{_line(text, m.start())}: stale repo name '{old}' (renamed to '{cfg['repo']}')") for m in PLACEHOLDER_RE.finditer(text): if m.group(0).lower() in allow: @@ -195,8 +207,7 @@ def check_rule_count(cfg: dict) -> list[str]: if not mod: return [] prefix = cfg["rule_prefix"] - ids = sorted(set(re.findall(rf"\b{prefix}\d+\b", - (ROOT / mod).read_text(encoding="utf-8")))) + ids = sorted(set(re.findall(rf"\b{prefix}\d+\b", (ROOT / mod).read_text(encoding="utf-8")))) n = len(ids) readme = (ROOT / cfg.get("docs", ["README.md"])[0]).read_text(encoding="utf-8") out: list[str] = [] @@ -224,7 +235,8 @@ def _score_cmd_re(modules: list[str]) -> re.Pattern: alt = "|".join(re.escape(m) for m in modules) return re.compile( r"(?Ppython3? +(?:-m +)?(?:" + alt + r")\b[^\n#]*?)" - r" *#[^\n]*?(?P\d+) */ *(?P100|10)\b") + r" *#[^\n]*?(?P\d+) */ *(?P100|10)\b" + ) def check_score_claims(cfg: dict) -> list[str]: @@ -249,24 +261,26 @@ def check_score_claims(cfg: dict) -> list[str]: argv = shlex.split(full) if not argv: continue - argv = [sys.executable] + argv[1:] + argv = [sys.executable, *argv[1:]] try: - proc = subprocess.run(argv, cwd=ROOT, capture_output=True, - text=True, timeout=300) + proc = subprocess.run(argv, cwd=ROOT, capture_output=True, text=True, timeout=300) except Exception as exc: # noqa: BLE001 - out.append(f"{d}:{_line(text, m.start())}: '{full}' did not run " - f"({type(exc).__name__})") + out.append(f"{d}:{_line(text, m.start())}: '{full}' did not run ({type(exc).__name__})") continue sm = re.search(rf"(\d+) */ *{denom}\b", proc.stdout) if not sm: err = (proc.stderr.strip().splitlines()[-1:] or [""])[0] - out.append(f"{d}:{_line(text, m.start())}: '{full}' documents " - f"{claimed}/{denom} but printed no score ({err[:80]})") + out.append( + f"{d}:{_line(text, m.start())}: '{full}' documents " + f"{claimed}/{denom} but printed no score ({err[:80]})" + ) continue actual = int(sm.group(1)) if actual != claimed: - out.append(f"{d}:{_line(text, m.start())}: '{full}' documents " - f"{claimed}/{denom} but the command scores {actual}/{denom}") + out.append( + f"{d}:{_line(text, m.start())}: '{full}' documents " + f"{claimed}/{denom} but the command scores {actual}/{denom}" + ) return out diff --git a/scripts/check_workload_proof_packets.py b/scripts/check_workload_proof_packets.py index b5d76f9..e4b0afc 100644 --- a/scripts/check_workload_proof_packets.py +++ b/scripts/check_workload_proof_packets.py @@ -16,7 +16,6 @@ from pathlib import Path from typing import Any - ROOT = Path(__file__).resolve().parents[1] SCHEMA_VERSION = "workload_proof_packet_v1" WORKFLOW_HANDOFF_SCHEMA_VERSION = "workflow_handoff_v1" diff --git a/scripts/extract-release-notes.py b/scripts/extract-release-notes.py index 7267aa3..c5eed02 100644 --- a/scripts/extract-release-notes.py +++ b/scripts/extract-release-notes.py @@ -7,11 +7,8 @@ import re import sys - VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)") -HEADING = re.compile( - rf"^## \[({VERSION.pattern})\] - ([0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}})$" -) +HEADING = re.compile(rf"^## \[({VERSION.pattern})\] - ([0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}})$") def main() -> int: diff --git a/scripts/freshness-report.py b/scripts/freshness-report.py index 5a5e5de..cd52211 100644 --- a/scripts/freshness-report.py +++ b/scripts/freshness-report.py @@ -23,7 +23,7 @@ import argparse import re import sys -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] @@ -42,7 +42,7 @@ def _parse_frontmatter(text: str) -> dict[str, str]: if m: fm[m.group(1)] = m.group(2) continue - metadata_match = re.match(r'^\s{2}(last-validated)\s*:\s*(.+?)\s*$', line) + metadata_match = re.match(r"^\s{2}(last-validated)\s*:\s*(.+?)\s*$", line) if metadata_match: fm[metadata_match.group(1)] = metadata_match.group(2) return fm @@ -54,7 +54,7 @@ def main() -> int: parser.add_argument("--red-days", type=int, default=180) args = parser.parse_args() - today = datetime.now(timezone.utc).date() + today = datetime.now(UTC).date() rows: list[tuple[int, str, str, bool]] = [] # (days, skill, date_str, is_adapted) for skill_md in sorted(SKILLS_DIR.glob("*/SKILL.md")): text = skill_md.read_text() @@ -100,7 +100,7 @@ def main() -> int: print(f"## Summary: {green_count} GREEN / {yellow_count} YELLOW / {red_count} RED / total {len(rows)}") print() print(f"Thresholds: YELLOW > {args.yellow_days}d, RED > {args.red_days}d.") - print(f"Adapted skills get RED -> WARN (refresh from upstream).") + print("Adapted skills get RED -> WARN (refresh from upstream).") if red_count > 0: return 2 diff --git a/scripts/lint-skill-counts.py b/scripts/lint-skill-counts.py index 39331d8..190c3d5 100644 --- a/scripts/lint-skill-counts.py +++ b/scripts/lint-skill-counts.py @@ -80,6 +80,7 @@ "forty-four": 44, } + def count_on_disk() -> int: """Return the on-disk installable skill count.""" if not SKILLS_DIR.is_dir(): @@ -112,26 +113,21 @@ def lint_doc(path: Path, expected: int) -> list[str]: if num == expected or num in seen_nums: continue seen_nums.add(num) - findings.append( - f"{path}:{line_no}: " - f"reports {num} skills (expected {expected}): {line.strip()[:160]}" - ) + findings.append(f"{path}:{line_no}: reports {num} skills (expected {expected}): {line.strip()[:160]}") for m in SPELLED_OUT_PATTERN.finditer(line): num = WORD_TO_INT[m.group(1).lower()] if num == expected or num in seen_nums: continue seen_nums.add(num) findings.append( - f"{path}:{line_no}: " - f"reports {m.group(0)} ({num}) skills (expected {expected}): {line.strip()[:160]}" + f"{path}:{line_no}: reports {m.group(0)} ({num}) skills (expected {expected}): {line.strip()[:160]}" ) return findings def main() -> int: expected = count_on_disk() - print(f"[lint-skill-counts] on-disk count: {expected} skills " - f"(under {SKILLS_DIR.relative_to(REPO_ROOT)})") + print(f"[lint-skill-counts] on-disk count: {expected} skills (under {SKILLS_DIR.relative_to(REPO_ROOT)})") all_findings: list[str] = [] for rel in DOCS_TO_LINT: all_findings.extend(lint_doc(REPO_ROOT / rel, expected)) diff --git a/scripts/lint-skill-mcp-args.py b/scripts/lint-skill-mcp-args.py index f7deaeb..bee2c4b 100644 --- a/scripts/lint-skill-mcp-args.py +++ b/scripts/lint-skill-mcp-args.py @@ -38,7 +38,6 @@ import subprocess import sys from pathlib import Path -from typing import Optional REPO_ROOT = Path(__file__).resolve().parents[1] SKILLS_DIR = REPO_ROOT / "plugins" / "profile-and-optimize" / "skills" @@ -76,6 +75,7 @@ "zymtrace": "user-zymtrace", } + def optional_servers() -> set[str]: """Servers that frontmatter `allowed-tools` MAY reference even when they are NOT declared in `.mcp.json`. @@ -110,11 +110,11 @@ def iter_skill_md_files() -> list[Path]: return sorted(SKILLS_DIR.glob("*/SKILL.md")) -def remap_server(server: str) -> Optional[str]: +def remap_server(server: str) -> str | None: return SERVER_REMAP.get(server) -def list_profile_and_optimize_surface() -> Optional[set[str]]: +def list_profile_and_optimize_surface() -> set[str] | None: """Use mcp_surface.py to enumerate the bundled profile_and_optimize MCP tool names.""" surface_py = SERVER_DIR / "mcp_surface.py" if not surface_py.exists(): @@ -124,7 +124,10 @@ def list_profile_and_optimize_surface() -> Optional[set[str]]: try: out = subprocess.run( [py, str(surface_py), "list"], - capture_output=True, text=True, check=True, cwd=str(SERVER_DIR), + capture_output=True, + text=True, + check=True, + cwd=str(SERVER_DIR), ).stdout except subprocess.CalledProcessError: return None @@ -144,14 +147,12 @@ def list_profile_and_optimize_surface() -> Optional[set[str]]: return names -def load_descriptor(server: str, tool: str) -> Optional[dict]: +def load_descriptor(server: str, tool: str) -> dict | None: remapped = remap_server(server) if remapped is None or not re.fullmatch(r"[A-Za-z0-9_]+", tool): return None root = os.path.realpath(os.fspath(MCP_CACHE)) - desc_path_text = os.path.realpath( - os.path.join(root, remapped, "tools", f"{tool}.json") - ) + desc_path_text = os.path.realpath(os.path.join(root, remapped, "tools", f"{tool}.json")) prefix = root if root.endswith(os.sep) else root + os.sep if not desc_path_text.startswith(prefix): return None @@ -217,7 +218,7 @@ def parse_frontmatter(text: str) -> dict: return {} fm_text = text[4:end] fm: dict = {} - cur_key: Optional[str] = None + cur_key: str | None = None for line in fm_text.splitlines(): if not line.strip(): continue @@ -234,7 +235,7 @@ def parse_frontmatter(text: str) -> dict: return fm -def lint_skill(skill_md: Path, profile_and_optimize_surface: Optional[set[str]]) -> list[dict]: +def lint_skill(skill_md: Path, profile_and_optimize_surface: set[str] | None) -> list[dict]: text = skill_md.read_text() fm = parse_frontmatter(text) body_start = text.find("\n---\n", 4) @@ -261,11 +262,23 @@ def lint_skill(skill_md: Path, profile_and_optimize_surface: Optional[set[str]]) continue if server == "profile_and_optimize": if profile_and_optimize_surface is not None and tool not in profile_and_optimize_surface: - findings.append({"severity": "RED", "skill": skill_name, "msg": f"frontmatter references mcp__profile_and_optimize__{tool}; tool not in bundled MCP surface (run `make mcp-surface` to inspect)"}) + findings.append( + { + "severity": "RED", + "skill": skill_name, + "msg": f"frontmatter references mcp__profile_and_optimize__{tool}; tool not in bundled MCP surface (run `make mcp-surface` to inspect)", + } + ) continue desc = load_descriptor(server, tool) if desc is None: - findings.append({"severity": "RED", "skill": skill_name, "msg": f"frontmatter references mcp__{server}__{tool}; descriptor JSON not found in {MCP_CACHE / (remap_server(server) or server) / 'tools' / (tool + '.json')}"}) + findings.append( + { + "severity": "RED", + "skill": skill_name, + "msg": f"frontmatter references mcp__{server}__{tool}; descriptor JSON not found in {MCP_CACHE / (remap_server(server) or server) / 'tools' / (tool + '.json')}", + } + ) # 2. Validate body `with:` arg names against the descriptor schema. for server, tool, args, lineno in extract_with_blocks(body): @@ -283,12 +296,14 @@ def lint_skill(skill_md: Path, profile_and_optimize_surface: Optional[set[str]]) valid_args = set(props.keys()) for arg in args: if arg not in valid_args: - findings.append({ - "severity": "RED", - "skill": skill_name, - "line": lineno, - "msg": f"body line {lineno}: mcp__{server}__{tool} with: '{arg}'; valid args are {sorted(valid_args)}", - }) + findings.append( + { + "severity": "RED", + "skill": skill_name, + "line": lineno, + "msg": f"body line {lineno}: mcp__{server}__{tool} with: '{arg}'; valid args are {sorted(valid_args)}", + } + ) return findings @@ -298,14 +313,20 @@ def main() -> int: return 2 if not MCP_CACHE.exists(): print(f"FATAL: MCP descriptor folder not found at {MCP_CACHE}", file=sys.stderr) - print(f" Bundled fallback expected at {BUNDLED_DESCRIPTORS} (see mcp-descriptors/README.md).", file=sys.stderr) + print( + f" Bundled fallback expected at {BUNDLED_DESCRIPTORS} (see mcp-descriptors/README.md).", + file=sys.stderr, + ) return 2 if MCP_CACHE == BUNDLED_DESCRIPTORS: print(f"[info] using bundled MCP descriptors at {MCP_CACHE} (no Cursor cache found)") profile_and_optimize_surface = list_profile_and_optimize_surface() if profile_and_optimize_surface is None: - print(f"WARN: could not load bundled profile_and_optimize MCP surface (run `bash {SERVER_DIR}/install.sh --with-dev`)", file=sys.stderr) + print( + f"WARN: could not load bundled profile_and_optimize MCP surface (run `bash {SERVER_DIR}/install.sh --with-dev`)", + file=sys.stderr, + ) red_total = 0 warn_total = 0 diff --git a/scripts/lint-tool-counts.py b/scripts/lint-tool-counts.py index 4dad861..80a6f9f 100644 --- a/scripts/lint-tool-counts.py +++ b/scripts/lint-tool-counts.py @@ -125,6 +125,25 @@ ) +def _report_mismatch( + findings: list[str], + path: Path, + line_no: int, + line: str, + seen_pairs: set[tuple[str, int]], + kind: str, + num: int, + expected: int, +) -> None: + if num == expected: + return + key = (kind, num) + if key in seen_pairs: + return + seen_pairs.add(key) + findings.append(f"{path}:{line_no}: reports {num} {kind} (expected {expected}): {line.strip()[:160]}") + + def lint_doc(path: Path, *, expected_contract: int, expected_total: int, expected_libs: int) -> list[str]: findings: list[str] = [] if not path.is_file(): @@ -136,27 +155,42 @@ def lint_doc(path: Path, *, expected_contract: int, expected_total: int, expecte seen_pairs: set[tuple[str, int]] = set() - def report(kind: str, num: int, expected: int) -> None: - if num == expected: - return - key = (kind, num) - if key in seen_pairs: - return - seen_pairs.add(key) - findings.append( - f"{path}:{line_no}: reports {num} {kind} (expected {expected}): " - f"{line.strip()[:160]}" - ) - for pat in CONTRACT_TOOL_PATTERNS: for m in pat.finditer(line): - report("contract-tools", int(m.group(1)), expected_contract) + _report_mismatch( + findings, + path, + line_no, + line, + seen_pairs, + "contract-tools", + int(m.group(1)), + expected_contract, + ) for pat in TOTAL_TOOL_PATTERNS: for m in pat.finditer(line): - report("MCP-tools-total", int(m.group(1)), expected_total) + _report_mismatch( + findings, + path, + line_no, + line, + seen_pairs, + "MCP-tools-total", + int(m.group(1)), + expected_total, + ) for pat in LIBRARY_PATTERNS: for m in pat.finditer(line): - report("libraries", int(m.group(1)), expected_libs) + _report_mismatch( + findings, + path, + line_no, + line, + seen_pairs, + "libraries", + int(m.group(1)), + expected_libs, + ) return findings diff --git a/scripts/lint-versions.py b/scripts/lint-versions.py index 3bf74f7..20c6e30 100644 --- a/scripts/lint-versions.py +++ b/scripts/lint-versions.py @@ -76,8 +76,7 @@ def release_version() -> str: version = raw.removesuffix("\n") if not NUMERIC_SEMVER.fullmatch(version): print( - f"FATAL: {VERSION_FILE} must contain numeric SemVer X.Y.Z " - "with at most one final newline", + f"FATAL: {VERSION_FILE} must contain numeric SemVer X.Y.Z with at most one final newline", file=sys.stderr, ) sys.exit(2) diff --git a/scripts/print-cursor-mcp-snippet.sh b/scripts/print-cursor-mcp-snippet.sh index ee778bd..fd9b7e3 100644 --- a/scripts/print-cursor-mcp-snippet.sh +++ b/scripts/print-cursor-mcp-snippet.sh @@ -16,6 +16,8 @@ SERVER_ROOT="${REPO_ROOT}/plugins/profile-and-optimize/server" VENV_PY="${SERVER_ROOT}/.venv/bin/python" if [[ ! -x "${VENV_PY}" ]]; then + # The backticks are literal command formatting in the warning. + # shellcheck disable=SC2016 printf '[warn] %s not found; run `make bootstrap` (or `bash %s/install.sh`) first.\n' \ "${VENV_PY}" "${SERVER_ROOT}" >&2 fi diff --git a/scripts/smoke-mcp-runtime.sh b/scripts/smoke-mcp-runtime.sh index 0264ca3..a7444b0 100644 --- a/scripts/smoke-mcp-runtime.sh +++ b/scripts/smoke-mcp-runtime.sh @@ -22,6 +22,7 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +PROJECT_VERSION="$(<"${REPO_ROOT}/VERSION")" SERVER="" # Default expected tool count is read at runtime from @@ -118,7 +119,7 @@ log "[1/4] expected tools: ${EXPECTED_TOOLS} (from mcp_surface.py canonical-coun # Run the round-trip via an embedded Python program. Sending raw JSON-RPC over # stdio works regardless of which MCP client we're invoking; the FastMCP server # implements the MCP wire protocol. -SMOKE_OUTPUT="$(PROFILE_AND_OPTIMIZE_REPO_ROOT="${SERVER}" "${VENV_PY}" - <<'PYEOF' +SMOKE_OUTPUT="$(PROFILE_AND_OPTIMIZE_REPO_ROOT="${SERVER}" PROJECT_VERSION="${PROJECT_VERSION}" "${VENV_PY}" - <<'PYEOF' import json import os import subprocess @@ -163,7 +164,7 @@ try: init_resp = send_request("initialize", { "protocolVersion": "2024-11-05", "capabilities": {}, - "clientInfo": {"name": "profile-and-optimize-smoke", "version": "0.2.1"}, + "clientInfo": {"name": "profile-and-optimize-smoke", "version": os.environ["PROJECT_VERSION"]}, }, req_id=1) if "error" in init_resp: result["errors"].append({"step": "initialize", "error": init_resp["error"]}) diff --git a/scripts/test_release_gates.py b/scripts/test_release_gates.py index 4fc9ed5..716525b 100644 --- a/scripts/test_release_gates.py +++ b/scripts/test_release_gates.py @@ -9,7 +9,6 @@ import unittest from pathlib import Path - REPO_ROOT = Path(__file__).resolve().parent.parent TRANSITION = REPO_ROOT / "scripts" / "check-version-transition.py" TAG_GATE = REPO_ROOT / "scripts" / "check-release-tag.sh" @@ -24,8 +23,7 @@ def run(*args: str, cwd: Path, stdin: str | None = None) -> subprocess.Completed cwd=cwd, input=stdin, text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=False, ) diff --git a/scripts/validate-cached.sh b/scripts/validate-cached.sh index d0368c9..12f0f4e 100644 --- a/scripts/validate-cached.sh +++ b/scripts/validate-cached.sh @@ -19,6 +19,9 @@ set -euo pipefail +# Single-quoted awk code and backticked command examples below are literal. +# shellcheck disable=SC2016 + SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" @@ -43,6 +46,8 @@ CACHE_FILE="${CACHE_DIR}/plugin-validate-${SHA}.ok" if [[ -f "${CACHE_FILE}" ]]; then TS="$(head -1 "${CACHE_FILE}" 2>/dev/null || echo 'unknown')" + # The backticks are literal command formatting in the status line. + # shellcheck disable=SC2016 printf '[cached] claude plugin validate (manifest sha=%s; validated at %s; bypass with `make validate-uncached`)\n' \ "${SHA}" "${TS}" exit 0 @@ -72,8 +77,11 @@ elif command -v claude >/dev/null 2>&1; then exit "${exit_code}" fi else + # The backticks are literal command formatting in these instructions. + # shellcheck disable=SC2016 printf 'FATAL: `claude` CLI not on PATH and CLAUDE_PLUGIN_VALIDATE_CMD unset.\n' >&2 printf ' Install Claude Code or export CLAUDE_PLUGIN_VALIDATE_CMD to a substitute (e.g.\n' >&2 + # shellcheck disable=SC2016 printf ' `python3 -m json.tool < %s/.claude-plugin/plugin.json`).\n' "${PLUGIN_DIR}" >&2 exit 2 fi diff --git a/scripts/validate-skill-prompt.sh b/scripts/validate-skill-prompt.sh index b15f2d3..463730f 100644 --- a/scripts/validate-skill-prompt.sh +++ b/scripts/validate-skill-prompt.sh @@ -12,6 +12,7 @@ REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" SERVER="${REPO_ROOT}/plugins/profile-and-optimize/server" VENV_PY="${SERVER}/.venv/bin/python" OUT_DIR="${OUT_DIR:-/tmp/profile-and-optimize-validate}" +PROJECT_VERSION="$(<"${REPO_ROOT}/VERSION")" mkdir -p "${OUT_DIR}" @@ -20,7 +21,8 @@ if [[ ! -x "${VENV_PY}" ]]; then exit 2 fi -PROFILE_AND_OPTIMIZE_REPO_ROOT="${SERVER}" "${VENV_PY}" - "${OUT_DIR}" "${SERVER}" <<'PYEOF' +PROFILE_AND_OPTIMIZE_REPO_ROOT="${SERVER}" PROJECT_VERSION="${PROJECT_VERSION}" \ + "${VENV_PY}" - "${OUT_DIR}" "${SERVER}" <<'PYEOF' """Skill→prompt simulation via stdio MCP server. For each skill, drive the first MCP call its workflow specifies, and record @@ -78,7 +80,7 @@ def call_tool(name, arguments): return None, resp -rpc("initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "profile-and-optimize-skill-validate", "version": "0.2.1"}}) +rpc("initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "profile-and-optimize-skill-validate", "version": os.environ["PROJECT_VERSION"]}}) notify("notifications/initialized", {})