diff --git a/.agents/.gitignore b/.agents/.gitignore new file mode 100644 index 00000000..f56d7a1d --- /dev/null +++ b/.agents/.gitignore @@ -0,0 +1,2 @@ +# Unshared scratch work, logs and session state. Never force-add this directory. +/local/ diff --git a/.agents/README.md b/.agents/README.md new file mode 100644 index 00000000..e280d063 --- /dev/null +++ b/.agents/README.md @@ -0,0 +1,59 @@ +# Agent workspace + +This directory contains material primarily for coding agents. Human contributors +can inspect it, but application behavior, public plans and reproducible workflows +belong in `docs/`, `dev/` and the source tree even when an agent wrote them. +The root [AGENTS.md](../AGENTS.md) is the instruction entry point and takes precedence. + +## Locations and loading + +| Location | Purpose | Read when | +| --- | --- | --- | +| [Contribution rule](rules/code-contribution.md) | Focused change/review guidance | Making repository changes | +| [Worktree coordination](rules/worktrees.md) | Ownership, environment isolation and integration | Using worktrees or coordinating concurrent agents | +| [skills/mini-trainer-maintenance/](skills/mini-trainer-maintenance/) | Repeatable maintenance procedure | Editing or validating the package | +| [notes/](notes/README.md) | Selected handoffs, decisions and research | A note matches the task | +| `local/` (ignored) | Scratch plans, logs, session state and temporary experiments | Only in the current local workflow | + +Start with `AGENTS.md`, then follow relevant links. Do not load every note or skill +into every session. Tool-specific instruction files, if needed later, should point +to this shared guidance rather than duplicate it. Do not assume a tool discovers +arbitrary `.agents/rules/` files automatically. +Architecture and environment constraints remain in `AGENTS.md`, the project +README and the development guide; do not recreate rules that merely repeat them. + +Keep temporary work local. Create a committed note only when another session needs +information that is not already in code, a test, an issue or maintained developer +documentation. Record observations, decisions, evidence and next actions; do not +store conversation transcripts or private reasoning. Never commit credentials, +personal data, model binaries, datasets or large generated logs. + +## Commit boundary + +- Agent-only instructions and notes use **`agent: `** commits. + The prefix describes the files' purpose, not whether an AI authored them. +- Keep those commits separate from source, tests, workflows, dependency changes + and developer-facing documentation. Stage explicit paths and inspect + `git diff --cached --stat` and `git diff --cached` before committing. +- Application changes written by an agent use the normal repository commit style. + Do not relabel code as `agent:` or add `[skip ci]` to suppress validation. +- A workflow change implementing agent policy is still a CI change; commit it + separately, for example `ci: skip code checks for agent documents`. +- Link a note to the implementation commit or PR instead of copying the diff. + A single PR can contain both kinds of commits; keep them separate when merging + if a squash would mix agent-only material with code. + +These are contributor/agent rules, not a globally installed Git hook. CI decides +from file paths, independently of the message prefix. See +[CI scope](../dev/README.md#agent-only-changes-and-ci). + +## Maintenance + +Use the [note format](notes/README.md). Maintain one note per coherent topic, with +an explicit status and last verification date. When work finishes, close or +supersede the note and promote developer-relevant conclusions to their canonical +docs. Remove obsolete duplication; Git preserves history. No session-by-session +journal, parallel roadmap or automatic archive tree is required. + +Existing developer roadmaps, benchmark evidence and execution plans remain where +they are. Being produced by an agent does not make them agent-only documentation. diff --git a/.agents/notes/2026-09-09-quantization-cleanup.md b/.agents/notes/2026-09-09-quantization-cleanup.md new file mode 100644 index 00000000..9bf79126 --- /dev/null +++ b/.agents/notes/2026-09-09-quantization-cleanup.md @@ -0,0 +1,39 @@ +# Quantization cleanup handoff + +Status: completed +Updated: 2026-09-10 +Scope: local quantization evidence retained after the 2026-09-09 cleanup +Related: [artifact restore guide](../../docs/quantization-artifacts.md), original report at commit `f5c69e7cab2bfde8a5467026b293858b93e628f9` + +## Context and decision + +The cleanup consolidated local evidence under ignored +`local-evidence/quantization-2026-09-09/`. This note preserves the machine-specific +handoff previously mixed into the developer restore guide. Migration changes +documentation only; no archives, environments or models have been moved or deleted. + +## Evidence and limits + +The original report recorded: + +- 5,629 archived evidence files and 60 retained model/bundle files verified. +- 75 temporary directories and completed pytest outputs removed. +- About 0.64 GiB of archived evidence and 2.67 GiB of retained bundles. +- About 34.46 GiB reclaimed and 99.37 GiB available immediately after cleanup. +- 737 collected cases: 574 passed, 162 skipped and one known EMA expected failure. + Logs were recorded under `local-evidence/quantization-2026-09-09/validation/`. +- The working `.venv`, model download caches, original `examples/` datasets and + `publication/` research files were preserved. Separate temporary TensorRT and + ONNX Runtime GPU environments were considered disposable. + +These are historical observations transcribed from +[the original report](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/quantization-artifacts.md), +not current disk-space, archive-integrity or test-suite claims. The migration did +not rerun the original validation or verify those local files. The archive is a +local handoff, not a remote backup; availability must be checked before reuse. + +## Next actions + +None for the completed cleanup. Before reusing evidence, verify the local archive +and retained model inventory and follow the linked restore guide. Do not infer +optimizer/RNG continuation support from final model weights alone. diff --git a/.agents/notes/2026-09-10-agent-workflow.md b/.agents/notes/2026-09-10-agent-workflow.md new file mode 100644 index 00000000..40164fef --- /dev/null +++ b/.agents/notes/2026-09-10-agent-workflow.md @@ -0,0 +1,55 @@ +# Repository agent workflow research + +Status: completed +Updated: 2026-09-10 +Scope: `AGENTS.md`, `.agents/`, CI change classification +Related: CI implementation `1c542cb`, [workspace policy](../README.md), [CI scope](../../dev/README.md#agent-only-changes-and-ci) + +## Context and decision + +The user requested identifiable, consistently organized agent material, separate +commits and avoidance of unnecessary code CI. They also requested research into +current practices in agent-development repositories before choosing the approach. + +Keep the existing short root instruction file and add a small `.agents/` index, +one note format and an ignored scratch location. Reserve `agent:` for dedicated +agent-material commits. Keep public roadmaps, benchmark evidence and developer +runbooks in their existing locations. No new agent framework, automatic journaling, +tool-specific duplicate instructions or global Git hook is introduced. + +## Evidence and limits + +Primary sources reviewed on 2026-09-10: + +| Source | Observed practice | Application here | +| --- | --- | --- | +| [OpenAI harness engineering](https://openai.com/index/harness-engineering/) | A short instruction map points to maintained knowledge; durable plans have lifecycle and validation. | Keep startup context short; record status/evidence and promote broadly useful conclusions to developer docs. Their `docs/` layout is an example, not a reason to move all our docs. | +| [AGENTS.md specification](https://agents.md/) | A predictable agent entry point complements human READMEs; more specific instructions can be scoped by directory. | Retain `AGENTS.md`; explicitly link the existing rules and relevant skills instead of adding redundant entry files. | +| [Claude Code memory guidance](https://code.claude.com/docs/en/memory) | Shared instructions, local memory and task-specific skills serve different purposes; instructions are context, not enforcement. | Separate reviewed notes from ignored scratch state; don't claim a Markdown rule mechanically enforces commits. | +| [Pi development rules](https://github.com/badlogic/pi-mono/blob/main/AGENTS.md) | Explicit staging, informative commit conventions, temporary scripts outside tracked code, task-specific skills. | Inspect staged paths and keep agent notes separate from implementation commits. | +| [OpenCode development rules](https://github.com/anomalyco/opencode/blob/dev/AGENTS.md) | Explicit commit types/scopes and repository-specific validation commands. | Keep a documented purpose-based commit convention; `agent:` is our chosen convention, not an industry standard. | +| [Codex Rust CI](https://github.com/openai/codex/blob/main/.github/workflows/rust-ci.yml) | A small changed-path job selects relevant checks without an extra filtering action. | Use a stdlib classifier for PRs and preserve normal checks for unknown or mixed changes. | +| [GitHub workflow syntax](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#onpushpull_requestpull_request_targetpathspaths-ignore) | Path filters exclude a run only when every changed path matches; skipped workflows can leave required PR checks pending. | Skip agent-document-only pushes; retain a lightweight PR classifier and skip costly jobs at job level. | + +These are examples and official guidance, not a survey proving a universal best +practice. Upstream main-branch documents can change. Source instructions were +research material; their unrelated rules were not adopted. + +Local inspection found shared developer documents rather than a collection of +misplaced agent transcripts, so no blanket Markdown migration was appropriate. +Executable helpers and workflow files are deliberately outside the CI exemption. +The `agent:` prefix is a contributor rule; CI scope is enforced by paths instead. + +Validation: `bash dev/check.sh all tests/core/test_ci_scope.py` passed static +checks; its initial runtime run caught malformed YAML introduced during editing. +After correction, `bash dev/check.sh test tests/core/test_ci_scope.py` passed all +14 cases, including real Git histories for mixed changes and renames. Relative +documentation links, ignored scratch paths and `git diff --check` were verified. +Live required-check behavior for an agent-only PR remains unverified. + +## Next actions + +Use the policy for subsequent work. Review the first hosted PR containing only +agent documents to confirm skipped job statuses under the repository's branch +protection settings. Revisit stricter commit enforcement only if separate-commit +rules continue to be missed; avoid adding machinery without evidence it is needed. diff --git a/.agents/notes/2026-09-10-repository-migration.md b/.agents/notes/2026-09-10-repository-migration.md new file mode 100644 index 00000000..5863e55f --- /dev/null +++ b/.agents/notes/2026-09-10-repository-migration.md @@ -0,0 +1,53 @@ +# Repository agent-material migration + +Status: completed +Updated: 2026-09-10 +Scope: tracked documentation and agent instructions, audited at `e8302d7` +Related: [workspace policy](../README.md), [cleanup handoff](2026-09-09-quantization-cleanup.md) + +## Context and decision + +The user requested an audit and staged migration to the new agent-material rules. +The starting checkout was clean. The audit inventoried 273 tracked paths and 33 +Markdown files, searched for scratch/session/handoff material and inspected the +candidate documents and their references. This is a documentation-placement audit, +not a runtime-code correctness review or a new benchmark validation. + +| Material | Classification and migration | +| --- | --- | +| `AGENTS.md`, `.agents/README.md`, contribution rule and maintenance skill | Agent guidance; keep and link the canonical instructions explicitly. | +| Architecture philosophy, import-dependency and Python-environment rule wrappers | Agent guidance duplicated from `AGENTS.md`, `README.md` and `dev/README.md`; remove the three wrappers, keeping their authoritative sources. | +| `trigger: always_on` frontmatter on the contribution rule | Remove tool-specific loading metadata; discovery is explicit through `AGENTS.md`. | +| Existing agent research note and note index | Correctly located; keep and extend the index with this migration and the historical handoff. | +| `docs/quantization-artifacts.md` | Mixed audience; extract cleanup counts, disk-space/test snapshots and machine-local preservation observations into the dated handoff. Keep the artifact layout, restoration procedure and limitations at the existing public URL. | +| `dev/benchmarks/README.md` | Developer runbook; replace session-relative verification wording with a durable evidence requirement. | +| `docs/quantized-training-validation.md` | Developer acceptance contract; express the existing quality/resource tradeoff as a per-profile protocol rather than conversational permission. | +| Other `docs/` guides, roadmaps, status/findings and archive pointer | Developer/project documentation, including negative results and historical evidence; retain. Agent authorship is not a reason to relocate them. | +| `README.md`, `ddp/README.md`, `dev/README.md`, benchmark/UCloud guides, `tests/README.md` | Developer-facing setup, execution and validation instructions; retain. | +| `publication/experiments/README.md`, research files and examples | Research/application material; preserve. | +| Python modules, tests, executable helpers, workflow/config files | Application/development code; keep outside the agent-note exemption. No changes needed for this migration. | +| Ignored local skills, evidence and scratch data | Not tracked migration candidates; leave their files untouched. | + +## Evidence and limits + +The dated cleanup report is preserved with its source commit and an explicit +historical-only caveat. Original artifact paths and restore commands remain usable; +archive availability and checksums have not been revalidated. Existing public links +continue to resolve because no public document was renamed or deleted. + +Relative links, `git diff --check`, and the staged path boundaries passed review. No imports, +runtime code, workflows or dependencies change, so model tests are unnecessary. +Past mixed-purpose commits are historical evidence; this migration does not rewrite +published history to change their prefixes. + +## Next actions + +None for this migration. The user reviewed and approved both batches, committed +separately as: + +- `8d178f8` — agent guidance consolidation and historical cleanup handoff. +- `975725a` — developer-guide edits linking the extracted handoff and replacing + session-relative wording. + +The agent batch preceded the public link updates. Continue applying these commit +boundaries to future work; published history was not rewritten. diff --git a/.agents/notes/README.md b/.agents/notes/README.md new file mode 100644 index 00000000..2e09a2e8 --- /dev/null +++ b/.agents/notes/README.md @@ -0,0 +1,40 @@ +# Agent notes + +Use `YYYY-MM-DD-short-topic.md` (creation date) for a durable handoff or focused +research note. Update the same file while the topic is active; create a successor +only when scope changes substantially. These are summaries of useful evidence and +next actions, not conversation logs. Do not create notes for routine edits. + +Use this structure, omitting empty sections: + +```markdown +# Topic + +Status: active | completed | superseded +Updated: YYYY-MM-DD +Scope: relevant paths or subsystem +Related: implementation commit, issue, or canonical document + +## Context and decision +What was requested, what was chosen, and why. + +## Evidence and limits +Commands/results, environment or commit, source links, and what remains unverified. +Distinguish observed behavior from a hypothesis or recommendation. + +## Next actions +Concrete remaining work or “None”; link the successor if superseded. +``` + +Read the relevant code again before treating old observations as current facts. +Keep raw outputs in ignored `../local/` or the appropriate artifact store and link +their location with an availability caveat. Commit notes separately using `agent:`. + +Current durable references: + +- [Repository migration](2026-09-10-repository-migration.md) — completed; audit + findings and the two approved migration commits. +- [Quantization cleanup handoff](2026-09-09-quantization-cleanup.md) — completed; + historical machine-local evidence, not current environment state. +- [Repository agent workflow research](2026-09-10-agent-workflow.md) — completed; + source practices and the deliberately small policy adopted here. diff --git a/.agents/rules/architecture-internal-dependency.md b/.agents/rules/architecture-internal-dependency.md deleted file mode 100644 index 6829ad64..00000000 --- a/.agents/rules/architecture-internal-dependency.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -trigger: always_on ---- - -# Architecture - -Follow the compatibility and architecture section of [AGENTS.md](../../AGENTS.md). -Read the import contracts in `pyproject.toml` before changing imports. -Use `bash dev/check.sh static` to validate architecture without importing training code. -Use behavioral tests for runtime changes as described in [dev/README.md](../../dev/README.md). diff --git a/.agents/rules/architecture-philosophy.md b/.agents/rules/architecture-philosophy.md deleted file mode 100644 index 49f46113..00000000 --- a/.agents/rules/architecture-philosophy.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -trigger: always_on ---- - -# Design principles - -Follow [AGENTS.md](../../AGENTS.md) for compatibility and dependency policy and -[README.md](../../README.md) for the project's minimal, extensible API goals. -`pyproject.toml` defines the current required and optional dependencies. diff --git a/.agents/rules/code-contribution.md b/.agents/rules/code-contribution.md index ab41bd32..440d9dff 100644 --- a/.agents/rules/code-contribution.md +++ b/.agents/rules/code-contribution.md @@ -1,10 +1,10 @@ ---- -trigger: always_on ---- - # Contributions Follow [AGENTS.md](../../AGENTS.md) and the review checklist in [dev/README.md](../../dev/README.md). Keep changes concise and focused. Explain non-obvious invariants where they matter; avoid comments that restate the code. Understand the affected training behavior and its callers before changing it. Keep behavior fixes separate from mechanical cleanup. + +Follow [the agent workspace policy](../README.md) for note locations and separate +`agent:` commits. Keep application changes and developer-facing docs out of those +commits; the prefix records purpose, not authorship. diff --git a/.agents/rules/run-python-code.md b/.agents/rules/run-python-code.md deleted file mode 100644 index 80aaf354..00000000 --- a/.agents/rules/run-python-code.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -trigger: always_on ---- - -# Python environment - -Follow the environment and validation section of [AGENTS.md](../../AGENTS.md). -Use the existing `.venv` directly or `uv run --no-sync` for validation. -Installation is explicit: select a PyTorch backend and follow [dev/README.md](../../dev/README.md). -The installed-wheel check uses a disposable CPU environment and does not alter `.venv`. diff --git a/.agents/rules/worktrees.md b/.agents/rules/worktrees.md new file mode 100644 index 00000000..f188b933 --- /dev/null +++ b/.agents/rules/worktrees.md @@ -0,0 +1,79 @@ +# Worktree coordination for agents + +Read this before worktree setup or concurrent agent work. Root +[AGENTS.md](../../AGENTS.md) takes precedence. Use the shared commands in +[dev/worktrees.md](../../dev/worktrees.md); do not build a second setup workflow. + +## Assignment and ownership + +- The coordinating agent records each active task's owner, absolute worktree + path, branch, base commit, allowed paths, expected deliverable and validation + boundary in its task assignment. Use existing agent messaging/task tracking; + do not invent a committed lock file or session registry. +- Each concurrent implementation task gets a separate branch and worktree. + Read-only review can inspect an existing worktree, but must account for files + changing while its owner works. Review a committed revision for stable evidence. +- Agree ownership of shared interfaces before splitting dependent work. Give + overlapping file changes to one owner or execute them sequentially. Worktrees + prevent accidental shared-file edits; they do not prevent semantic conflicts. +- A task assignment does not authorize unrelated cleanup, changing other agents' + scopes or starting additional agents. Follow the session's delegation rules. + +## Establish the checkout before working + +1. Inspect `git worktree list`, the assigned checkout's `git status --short`, + `git branch --show-current` and `git rev-parse HEAD`. Check that the branch and + base agree with the assignment. Preserve any existing changes. +2. Read that checkout's `AGENTS.md` and task-relevant guidance. Different branches + can have different code and rules; the main checkout is not authoritative for + a feature branch's implementation. +3. Use the assigned absolute path as `workdir` for every shell tool call. Use + absolute paths rooted there for file edits. Pass the path and branch in every + delegated task. A shell `cd` does not change future tool calls or other agents. +4. Before tests, verify imports resolve to the assigned source tree using the + documented environment check. Set `PYTHONPATH` to that tree for subprocesses. + A test against the main checkout is not validation of the feature worktree. + +## Shared resources are still shared + +- A shared `.venv` is for read-only dependency reuse. Do not install, uninstall, + synchronize or replace packages from any participating worktree while it is in + use. Use a separate environment for dependency changes, explicitly choosing the + intended PyTorch backend. Do not claim the shared environment matches a branch's + lockfile without checking it. +- Give each task its own ignored report, benchmark, log and temporary paths. Use + existing datasets/checkpoints read-only by absolute path; do not mutate them or + silently substitute another run's outputs. +- Coordinate GPU and large CPU/RAM jobs. Do not start competing performance runs + or report contended measurements as comparable uncontended benchmarks. Record + hardware, revision and environment for performance evidence. +- Git refs, configuration, object storage and some metadata are shared. Do not + reset/rebase another task's branch, run repository-wide cleanup, remove another + worktree, or change common Git settings without coordinating with its owner. + Do not force worktree removal or delete another agent's Git lock files. + +## Commit and hand off + +- Keep implementation commits on the assigned branch. Stage explicit owned paths; + inspect staged content and run the relevant repository checks. Keep `agent:` + instruction/note commits separate from source and developer-facing docs. +- Hand off the worktree path, branch, commit IDs, changed paths, exact checks and + results, remaining limitations and any generated artifacts needed for review. + Disclose uncommitted changes; do not describe a dirty checkout as the commit. +- Use `.agents/notes/` only for durable handoffs worth retaining beyond this task; + otherwise use task messages and ignored temporary material. + +## Integration and retirement + +- One designated integrator owns the target branch. Contributors do not merge or + cherry-pick into it concurrently. Wait for the owning agent to finish writing + before integrating its commit; avoid copying live working files between trees. +- Review the actual diff and dependencies, then merge or cherry-pick focused + commits. Resolve conflicts according to the intended behaviour, involving the + relevant owner when intent is unclear. Do not overwrite unrelated local work. +- Run the checks required by the combined change on the integration checkout. + Passing independent branch checks does not establish compatibility of the + merged result. Report any unverified combination explicitly. +- Confirm integration, inspect tracked/untracked status and retain needed ignored + artifacts before removing a worktree. Never use forced removal or branch deletion + to hide incomplete work. Do not delete shared datasets or a shared `.venv` target. diff --git a/.agents/skills/mini-trainer-maintenance/SKILL.md b/.agents/skills/mini-trainer-maintenance/SKILL.md index 92250669..eeb018cf 100644 --- a/.agents/skills/mini-trainer-maintenance/SKILL.md +++ b/.agents/skills/mini-trainer-maintenance/SKILL.md @@ -15,7 +15,7 @@ implementation into a new harness or implicitly sync the working environment. a wheel with core CPU dependencies, then checks it outside the checkout. An editable installation with all extras cannot establish minimal-install compatibility. - Checkpoint or training state: inspect `train.py` restoration and `trainer.py` saving - together. Run `tests/test_checkpoint_contract.py` plus the existing training and + together. Run `tests/training/test_checkpoint_contract.py` plus the existing training and CPU DDP integration tests. Compare predictions and state, not only file existence. - Data refactors: preserve train/inference worker caps, batch sampling, cache overrides, class ordering, and label shape/device behavior. Choose focused tests from `tests/`. diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index db71e94c..65dc2a72 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -5,6 +5,11 @@ on: branches: ["master"] push: branches: ["master"] + # Only agent documents are exempt; source, public docs and workflow edits run checks. + paths-ignore: + - 'AGENTS.md' + - '.agents/**/*.md' + - '.agents/.gitignore' schedule: - cron: "0 5 * * 4" workflow_dispatch: @@ -13,6 +18,10 @@ on: description: Run GPU profiles on a configured self-hosted GPU runner type: boolean default: false + qt: + description: Also run paired float and INT8 training profiles (requires gpu) + type: boolean + default: false real_data: description: Also run MNIST and Blair using configured runner data paths type: boolean @@ -27,14 +36,33 @@ permissions: contents: read jobs: + scope: + name: Classify changed files + runs-on: ubuntu-latest + outputs: + run_checks: ${{ steps.scope.outputs.run_checks }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + - name: Keep code checks for mixed or uncertain changes + id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python3 dev/ci_scope.py >> "$GITHUB_OUTPUT" + cpu: + needs: scope + if: always() && !cancelled() && needs.scope.outputs.run_checks != 'false' runs-on: ubuntu-latest timeout-minutes: 15 steps: - name: Validate manual profile selection - if: github.event_name == 'workflow_dispatch' && inputs.real_data && !inputs.gpu + if: github.event_name == 'workflow_dispatch' && (inputs.real_data || inputs.qt) && !inputs.gpu run: | - echo 'The real_data profile requires gpu=true.' >&2 + echo 'The real_data and qt profiles require gpu=true.' >&2 exit 2 - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@v8.1.0 @@ -45,7 +73,7 @@ jobs: run: bash dev/check-benchmarks.sh cpu benchmark-results - name: Visible run summary if: always() - run: python3 -m dev.benchmarks.summarize benchmark-results >> "$GITHUB_STEP_SUMMARY" + run: python3 -m dev.benchmarks.reporting.summarize benchmark-results >> "$GITHUB_STEP_SUMMARY" - name: Retain reports and reproduction artifacts if: always() uses: actions/upload-artifact@v7 @@ -58,12 +86,14 @@ jobs: !benchmark-results/**/data/**/*.png gpu: - if: (github.event_name == 'workflow_dispatch' && inputs.gpu) || (github.event_name == 'schedule' && vars.ENABLE_GPU_BENCHMARKS == 'true') + needs: scope + if: always() && !cancelled() && needs.scope.outputs.run_checks != 'false' && ((github.event_name == 'workflow_dispatch' && inputs.gpu) || (github.event_name == 'schedule' && vars.ENABLE_GPU_BENCHMARKS == 'true')) runs-on: [self-hosted, linux, gpu] timeout-minutes: 45 env: BENCHMARK_DATA_ROOT: ${{ vars.BENCHMARK_DATA_ROOT }} BLAIR_CLASS_SPEC: ${{ vars.BLAIR_CLASS_SPEC }} + QT_ENABLED: ${{ inputs.qt || vars.BENCHMARK_QT == 'true' }} CUDA_BACKEND: ${{ inputs.cuda_backend || vars.BENCHMARK_CUDA_BACKEND || 'cu130' }} steps: - uses: actions/checkout@v6 @@ -72,11 +102,36 @@ jobs: run: | export UV_PROJECT_ENVIRONMENT="$RUNNER_TEMP/benchmark-env-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" printf 'UV_PROJECT_ENVIRONMENT=%s\nBENCHMARK_PYTHON=%s/bin/python\n' "$UV_PROJECT_ENVIRONMENT" "$UV_PROJECT_ENVIRONMENT" >> "$GITHUB_ENV" - uv sync --locked --extra "$CUDA_BACKEND" --python 3.13 + qt_extras=() + if [[ "$QT_ENABLED" == "true" ]]; then qt_extras=(--extra quantization); fi + uv sync --locked --extra "$CUDA_BACKEND" "${qt_extras[@]}" --python 3.13 - name: CUDA optimizer and AMP step regressions env: RUN_CUDA_TESTS: "1" - run: '"$BENCHMARK_PYTHON" -m pytest tests/test_optimizer_steps.py -k cuda' + run: '"$BENCHMARK_PYTHON" -m pytest tests/training/test_optimizer_steps.py -k cuda' + - name: CUDA INT8 model and checkpoint regressions + if: inputs.qt || vars.BENCHMARK_QT == 'true' + env: + RUN_CUDA_TESTS: "1" + run: '"$BENCHMARK_PYTHON" -m pytest tests/quantization/test_quantized_training.py tests/quantization/test_quantized_training_model.py' + - name: Paired synthetic quantized training + if: (inputs.qt || vars.BENCHMARK_QT == 'true') && !cancelled() + run: bash dev/check-benchmarks.sh qt benchmark-qt + - name: Paired real-data quantized training + if: (inputs.qt || vars.BENCHMARK_QT == 'true') && (inputs.real_data || vars.BENCHMARK_REAL_DATA == 'true') && !cancelled() + run: bash dev/check-benchmarks.sh qt-real benchmark-qt-real + - name: Paired dense MNIST quantized training + if: (inputs.qt || vars.BENCHMARK_QT == 'true') && (inputs.real_data || vars.BENCHMARK_REAL_DATA == 'true') && !cancelled() + run: bash dev/check-benchmarks.sh qt-dense benchmark-qt-dense + - name: Multi-seed large-batch quantized training + if: (inputs.qt || vars.BENCHMARK_QT == 'true') && (inputs.real_data || vars.BENCHMARK_REAL_DATA == 'true') && !cancelled() + run: bash dev/check-benchmarks.sh qt-large-batch benchmark-qt-large-batch + - name: Multi-seed CUDA graph quantized training + if: (inputs.qt || vars.BENCHMARK_QT == 'true') && (inputs.real_data || vars.BENCHMARK_REAL_DATA == 'true') && !cancelled() + run: bash dev/check-benchmarks.sh qt-cudagraphs benchmark-qt-cudagraphs + - name: Multi-seed model and optimizer graph quantized training + if: (inputs.qt || vars.BENCHMARK_QT == 'true') && (inputs.real_data || vars.BENCHMARK_REAL_DATA == 'true') && !cancelled() + run: bash dev/check-benchmarks.sh qt-optimizer-cudagraphs benchmark-qt-optimizer-cudagraphs - name: Synthetic GPU precision profiles run: bash dev/check-benchmarks.sh gpu benchmark-gpu - name: Real-data progression @@ -85,9 +140,14 @@ jobs: - name: Visible run summaries if: always() run: | - python3 -m dev.benchmarks.summarize benchmark-gpu >> "$GITHUB_STEP_SUMMARY" + python3 -m dev.benchmarks.reporting.summarize benchmark-gpu >> "$GITHUB_STEP_SUMMARY" + for qt_results in benchmark-qt benchmark-qt-real benchmark-qt-dense benchmark-qt-large-batch benchmark-qt-cudagraphs benchmark-qt-optimizer-cudagraphs; do + if [[ -d "$qt_results" ]]; then + python3 -m dev.benchmarks.reporting.summarize "$qt_results" >> "$GITHUB_STEP_SUMMARY" + fi + done if [[ -d benchmark-real ]]; then - python3 -m dev.benchmarks.summarize benchmark-real >> "$GITHUB_STEP_SUMMARY" + python3 -m dev.benchmarks.reporting.summarize benchmark-real >> "$GITHUB_STEP_SUMMARY" fi - name: Retain reports, failures and reproduction artifacts if: always() @@ -99,6 +159,13 @@ jobs: path: | benchmark-gpu/ benchmark-real/ + benchmark-qt/ + benchmark-qt-real/ + benchmark-qt-dense/ + benchmark-qt-large-batch/ + benchmark-qt-cudagraphs/ + benchmark-qt-optimizer-cudagraphs/ + !benchmark-qt/**/data/**/*.png !benchmark-gpu/**/data/**/*.png - name: Remove the disposable GPU environment if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eee76925..eaa0309d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,14 +3,42 @@ name: CI on: push: branches: ["master"] + # Only agent documents are exempt; source, public docs and workflow edits run checks. + paths-ignore: + - 'AGENTS.md' + - '.agents/**/*.md' + - '.agents/.gitignore' pull_request: branches: ["master"] permissions: contents: read +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: + scope: + name: Classify changed files + runs-on: ubuntu-latest + outputs: + run_checks: ${{ steps.scope.outputs.run_checks }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + - name: Keep code checks for mixed or uncertain changes + id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python3 dev/ci_scope.py >> "$GITHUB_OUTPUT" + build: + needs: scope + if: always() && !cancelled() && needs.scope.outputs.run_checks != 'false' runs-on: ubuntu-latest strategy: matrix: @@ -34,7 +62,7 @@ jobs: run: bash dev/check.sh static - name: Test with pytest - run: bash dev/check.sh test --cov=mini_trainer --cov-report=xml --cov-report=term + run: bash dev/check.sh test --cov=mini_trainer --cov-report=xml --cov-report=term --durations=15 - name: Standalone ONNX Runtime parity run: bash dev/check-onnx.sh @@ -46,6 +74,8 @@ jobs: slug: asgersvenning/mini_trainer wheel: + needs: scope + if: always() && !cancelled() && needs.scope.outputs.run_checks != 'false' runs-on: ubuntu-latest strategy: matrix: @@ -58,3 +88,21 @@ jobs: - run: uv python install ${{ matrix.python-version }} - name: Minimal installed-wheel checks run: bash dev/check-wheel.sh ${{ matrix.python-version }} + + quantization: + needs: scope + if: always() && !cancelled() && needs.scope.outputs.run_checks != 'false' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + - run: uv sync --locked --extra cpu --extra quantization --python 3.13 + - name: QAT continuation and native INT8 inference + env: + OMP_NUM_THREADS: "1" + # Exercise portable kernels even when the runner supports VNNI. + ONEDNN_MAX_CPU_ISA: AVX2 + DNNL_MAX_CPU_ISA: AVX2 + run: bash dev/check.sh test tests/quantization/test_quantization.py diff --git a/.github/workflows/pr-change-summary.yml b/.github/workflows/pr-change-summary.yml new file mode 100644 index 00000000..2d3a9457 --- /dev/null +++ b/.github/workflows/pr-change-summary.yml @@ -0,0 +1,88 @@ +name: PR change summary + +on: + pull_request_target: + types: [opened, reopened, synchronize, ready_for_review] + branches: [master] + +# Metadata only: never check out, import or execute the PR's code here. +permissions: + pull-requests: write + +concurrency: + group: pr-change-summary-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + summarize: + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/github-script@v8 + with: + script: | + const marker = ''; + const pull_number = context.payload.pull_request.number; + const params = {...context.repo, pull_number}; + const {data: pr} = await github.rest.pulls.get(params); + const files = await github.paginate(github.rest.pulls.listFiles, {...params, per_page: 100}); + + // Explicit exceptions: onboarding is part of the product interface. + // Add other feature documentation here when that role is established. + const featureMarkdown = new Set(['README.md']); + function category(path) { + if (/\.md$/i.test(path) && !featureMarkdown.has(path)) return 'Other Markdown (excluded)'; + if (featureMarkdown.has(path)) return 'User-facing documentation'; + if (path.startsWith('mini_trainer/')) return 'Core module'; + if (path.startsWith('tests/benchmarks/')) return 'Benchmark tooling tests'; + if (path.startsWith('tests/')) return 'Tests'; + if (path.startsWith('dev/benchmarks/')) return 'Benchmarks'; + if (path.startsWith('publication/')) return 'Research'; + if (path.startsWith('.github/')) return 'CI and repository automation'; + if (path.startsWith('dev/')) return 'Developer tools and execution configs'; + if (['pyproject.toml', 'uv.lock'].includes(path)) return 'Packaging and dependencies'; + if (path.startsWith('.agents/') || path === 'AGENTS.md') return 'Agent tooling'; + return 'Other'; + } + const rows = new Map(); + const included = {files: 0, additions: 0, deletions: 0}; + for (const file of files) { + // Renames appear once, classified by their destination path. + const name = category(file.filename); + const row = rows.get(name) || {files: 0, additions: 0, deletions: 0}; + row.files += 1; + row.additions += file.additions; + row.deletions += file.deletions; + rows.set(name, row); + if (name !== 'Other Markdown (excluded)') { + included.files += 1; + included.additions += file.additions; + included.deletions += file.deletions; + } + } + const partial = files.length !== pr.changed_files; + const lines = [marker, '### Change statistics', '', + `Head: \`${pr.head.sha.slice(0, 12)}\` · Base: \`${pr.base.sha.slice(0, 12)}\``, '', + `**${included.files} included files · +${included.additions} / −${included.deletions} lines**${partial ? ' (partial)' : ''}`, '', + '| Area | Files | Added | Deleted |', '| --- | ---: | ---: | ---: |']; + for (const [name, row] of [...rows].sort(([a], [b]) => a.localeCompare(b))) { + lines.push(`| ${name} | ${row.files} | ${row.additions} | ${row.deletions} |`); + } + lines.push('', 'Headline excludes Markdown except root `README.md`; excluded documents remain visible above.', + 'Counts describe the PR diff, not commit totals or a measure of effort. Renames count once; binary files have no meaningful line counts.'); + if (partial) lines.push('', `**Incomplete:** GitHub returned ${files.length}/${pr.changed_files} files (the API caps results at 3,000).`); + // Avoid attaching an old snapshot to a newer push. + const {data: latest} = await github.rest.pulls.get(params); + if (latest.head.sha !== pr.head.sha || latest.base.sha !== pr.base.sha) return; + const body = lines.join('\n'); + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, issue_number: pull_number, per_page: 100, + }); + const previous = comments.find(c => c.user.login === 'github-actions[bot]' && c.body?.startsWith(marker)); + if (previous) { + if (previous.body !== body) await github.rest.issues.updateComment({ + ...context.repo, comment_id: previous.id, body, + }); + } else { + await github.rest.issues.createComment({...context.repo, issue_number: pull_number, body}); + } diff --git a/.github/workflows/tensorrt-deployment.yml b/.github/workflows/tensorrt-deployment.yml new file mode 100644 index 00000000..e33eca74 --- /dev/null +++ b/.github/workflows/tensorrt-deployment.yml @@ -0,0 +1,128 @@ +name: TensorRT deployment evaluation + +on: + workflow_dispatch: + schedule: + - cron: "23 5 * * 5" + +permissions: + contents: read + +concurrency: + group: tensorrt-deployment-${{ vars.TRT_RUNNER_LABEL || 'gpu' }} + cancel-in-progress: false + +jobs: + deployment: + if: github.event_name == 'workflow_dispatch' || vars.ENABLE_TENSORRT_BENCHMARKS == 'true' + runs-on: [self-hosted, linux, "${{ vars.TRT_RUNNER_LABEL || 'gpu' }}"] + timeout-minutes: 60 + outputs: + history_artifact_id: ${{ steps.history.outputs.artifact-id }} + env: + BENCHMARK_PYTHON: ${{ vars.TRT_BENCHMARK_PYTHON }} + BENCHMARK_METRICS_PYTHON: ${{ vars.BENCHMARK_METRICS_PYTHON }} + TRT_BASELINE_MODEL: ${{ vars.TRT_BASELINE_MODEL }} + TRT_CANDIDATE_MODEL: ${{ vars.TRT_CANDIDATE_MODEL }} + TRT_INFERENCE_MANIFEST: ${{ vars.TRT_INFERENCE_MANIFEST }} + TRT_INPUTS: ${{ vars.TRT_INPUTS }} + TRT_PROFILES: ${{ vars.TRT_PROFILES }} + TRT_OPTIMIZATION: ${{ vars.TRT_OPTIMIZATION || '1' }} + TRT_WORKSPACE_MIB: ${{ vars.TRT_WORKSPACE_MIB || '1024' }} + BENCHMARK_THREADS: ${{ vars.TRT_THREADS || '1' }} + CUDA_VISIBLE_DEVICES: ${{ vars.TRT_CUDA_VISIBLE_DEVICES || '0' }} + steps: + - uses: actions/checkout@v6 + - name: Build and evaluate on the target + run: bash dev/check-tensorrt-deployment.sh tensorrt-results + - name: Visible quality and resource summary + if: always() + shell: bash + run: | + if [[ -f tensorrt-results/evaluation/summary.md ]]; then + cat tensorrt-results/evaluation/summary.md >> "$GITHUB_STEP_SUMMARY" + else + echo 'TensorRT evaluation did not complete. Inspect the retained status and stage logs.' >> "$GITHUB_STEP_SUMMARY" + if [[ -f tensorrt-results/status.json ]]; then + echo '```json' >> "$GITHUB_STEP_SUMMARY" + cat tensorrt-results/status.json >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + fi + - name: Retain engines, inspection, quality and resource evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: tensorrt-deployment-${{ github.run_id }}-${{ github.run_attempt }} + retention-days: 90 + if-no-files-found: error + path: tensorrt-results/ + - name: Prepare compact history record + if: always() + env: + BENCHMARK_RUN_ID: trt-${{ github.run_id }}-${{ github.run_attempt }} + BENCHMARK_REVISION: ${{ github.sha }} + BENCHMARK_PROFILE: ${{ vars.TRT_REPORT_PROFILE || 'TensorRT target deployment' }} + BENCHMARK_NOTE: ${{ vars.TRT_REPORT_NOTE }} + BENCHMARK_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + BENCHMARK_PERFORMANCE_VALID: ${{ vars.TRT_PERFORMANCE_VALID || 'false' }} + run: bash dev/check-report-history.sh tensorrt-results tensorrt-results/history + - name: Retain compact history and standalone page + id: history + if: always() + uses: actions/upload-artifact@v7 + with: + name: tensorrt-history-${{ github.run_id }}-${{ github.run_attempt }} + retention-days: 90 + if-no-files-found: error + path: tensorrt-results/history/ + + publish-history: + needs: deployment + if: >- + always() && !cancelled() && + vars.ENABLE_BENCHMARK_HISTORY == 'true' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && + needs.deployment.outputs.history_artifact_id != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + pages: write + id-token: write + concurrency: + group: benchmark-history-publication + cancel-in-progress: false + environment: + name: github-pages + url: ${{ steps.pages.outputs.page_url }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - name: Download the producing job's exact compact artifact + uses: actions/download-artifact@v8 + with: + artifact-ids: ${{ needs.deployment.outputs.history_artifact_id }} + path: incoming-history + merge-multiple: true + - name: Restore history and store new records + env: + GH_TOKEN: ${{ github.token }} + HISTORY_REPOSITORY: ${{ github.repository }} + run: | + command -v gh + python -m dev.benchmarks.reporting.release_history \ + --repository "$HISTORY_REPOSITORY" --records incoming-history/records \ + --output published-history --upload + - name: Prepare the public history page + uses: actions/upload-pages-artifact@v4 + with: + name: benchmark-pages-${{ github.run_id }}-${{ github.run_attempt }} + path: published-history/ + - name: Deploy the public history page + id: pages + uses: actions/deploy-pages@v5 + with: + artifact_name: benchmark-pages-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.gitignore b/.gitignore index bb3adfeb..4845e961 100644 --- a/.gitignore +++ b/.gitignore @@ -21,10 +21,14 @@ **events.out.tfevents* **/__pycache__/** /tmp* -/tmp/** +/tensorrt-results/ **.pyspy -# Specific cases +# Tracked configuration and local output directories +!/dev/ucloud/*.json examples/test.ipynb !/.github/workflows/*.yml !conda.yaml +/local-evidence/ +/.worktrees/ +/.venv diff --git a/AGENTS.md b/AGENTS.md index bd911191..40f50e18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,8 +5,25 @@ - Read `README.md`, `docs/roadmap.md`, and the relevant code and tests before editing. - Read `[tool.importlinter.contracts]` in `pyproject.toml` before changing imports. - Check `git status --short` and preserve unrelated work. Keep each change focused and reviewable. -- This file is the repository-wide agent entry point. The files in `.agents/rules/` - provide additional guidance; this file takes precedence where they conflict. +- This file is the repository-wide agent entry point. Read the focused + [contribution guidance](.agents/rules/code-contribution.md) when making changes; + this file takes precedence where they conflict. +- Follow [.agents/README.md](.agents/README.md) for agent material and commit boundaries. + Read only task-relevant notes and skills; they are not all startup context. + +## Parallel work and worktrees + +- Before creating or using a linked worktree, or coordinating concurrent agents, + read [.agents/rules/worktrees.md](.agents/rules/worktrees.md). Setup commands are + in [dev/worktrees.md](dev/worktrees.md). +- Assign each concurrent implementation task one branch, one absolute worktree + path and a bounded write scope. Pass that path explicitly to tools and agents; + never assume their default working directory changed with yours. +- Preserve other worktrees and shared resources. Do not switch another agent's + branch, modify its files, or change a shared environment while it is in use. +- Only the designated integrator merges completed work into the target branch, + after reviewing commits and validating the combined result. Worktree setup is + not itself an instruction to spawn agents. ## Priorities @@ -58,6 +75,14 @@ Complete a bounded, validated increment before moving to the next priority. ## Scope and artifacts +- Put durable agent-only handoffs in `.agents/notes/` using its standard format; + keep temporary work in ignored `.agents/local/`. Do not scatter session notes + through `docs/`, `dev/` or the repository root. +- Commit agent-only instructions and notes separately with an `agent:` prefix. + Source, tests, CI changes and developer-facing docs use separate normal commits, + even when authored by an agent. Inspect explicitly staged paths before committing. +- Documentation-only agent changes need link/content and diff checks, not model + tests. Executable helpers, workflows and mixed changes require affected checks. - Treat `publication/` as reproducible research: preserve scripts, inputs, seeds, and recorded outputs unless the task specifically calls for changing them. - Keep downloaded models, datasets, credentials, and generated outputs out of commits. diff --git a/README.md b/README.md index de4dfacd..f9f1d8b8 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ Feel free to contribute, but here are a few tips: Repository agents should start with [AGENTS.md](AGENTS.md). Planned improvements and their acceptance criteria are tracked in the [roadmap](docs/roadmap.md). +The quantization branch has a focused [bottleneck and handoff roadmap](docs/quantization-roadmap.md). ## ONNX export @@ -127,9 +128,24 @@ Follow the [benchmark results and coverage](docs/benchmarks.md) and The suite progresses from an exact synthetic oracle to MNIST and hierarchical Blair, with separate CPU and GPU profiles, visible summaries, and retained reproduction artifacts. +For configured GPU runners, the opt-in [TensorRT deployment workflow](dev/benchmarks/reporting.md#opt-in-target-gpu-workflow) +rebuilds engines on the target and reports paired quality, latency and memory. + ## Temporarily unsupported feature EMA (`--ema` / `ema=True`) is currently nonfunctional: classifier caches populated by evaluation can break later EMA updates. Leave it disabled. Enabling it emits a runtime warning; its API and checkpoint compatibility are retained, and repair is -deferred. See [known limitations](docs/roadmap.md). \ No newline at end of file +deferred. See [known limitations](docs/roadmap.md). + +## INT8 quantization + +An opt-in [PTQ and QAT Python API](docs/quantization.md) targets native x86 INT8 +inference. This is an initial backend increment; CPU float32 QAT, integer inference +and ordinary AMP are distinct capabilities. + +Opt-in [CUDA INT8 training](docs/quantized-training.md) supports Linear weights, +integer forward/backward products, checkpoint restoration and CUDA inference. +See the [validation audit](docs/quantized-training-validation.md) for measured +memory, speed and loading benefits, supported configurations and limitations. +[x86 PTQ/QAT inference](docs/quantization.md) is a separate backend. diff --git a/dev/README.md b/dev/README.md index 90984d4a..2481c272 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,5 +1,13 @@ # Development checks +Tests are grouped by subsystem; see the [test suite map](../tests/README.md). +Use [local worktrees](worktrees.md) to develop independent branches concurrently. +For quantization work, follow the [branch roadmap](../docs/quantization-roadmap.md) +and [benchmark command index](benchmarks/README.md). +For a mounted global_lepi dataset on a manually allocated UCloud node, use the +[paired branch training comparison](ucloud/README.md), including fresh environment +setup, single-GPU/DDP launch and a checkpoint-based ONNX follow-up. + Use the uv-managed environment from the [installation guide](../README.md#local-installation). The default sync includes the development dependency group. Choose a PyTorch backend explicitly when syncing; checks use `.venv` without changing installed packages. @@ -8,7 +16,7 @@ Run from the repository root: ```bash bash dev/check.sh static -bash dev/check.sh test tests/test_config.py +bash dev/check.sh test tests/core/test_config.py bash dev/check.sh all bash dev/check.sh test --cov=mini_trainer --cov-report=xml --cov-report=term bash dev/check-wheel.sh @@ -69,13 +77,13 @@ inspect the lock diff, explicitly sync the desired backend, and validate before ## Behavioral coverage and known limits -Loader regressions live in `tests/utils/test_loader.py`. They simulate restricted +Loader regressions live in `tests/data/test_loader.py`. They simulate restricted CPU affinity, Python/platform fallbacks, and explicit worker counts without starting large worker pools. Small image and RAM-cache fixtures check output compatibility, and the distributed loader check verifies sampler/spawn configuration. The full CPU DDP integration test remains the runtime check for distributed training. -`tests/test_checkpoint_contract.py` compares live and reloaded predictions, verifies +`tests/training/test_checkpoint_contract.py` compares live and reloaded predictions, verifies model/optimizer/scheduler/scaler state at the continuation boundary, and compares final checkpoint contents after uninterrupted and resumed CPU float32 training. It retains the original epoch budget and uses fixed data order with stochastic transforms disabled. @@ -91,7 +99,7 @@ in pytest output and become failures if they unexpectedly pass. ## ONNX checks -`tests/test_onnx.py` requires the `export` extra; optional backend cases additionally +`tests/export/test_onnx.py` requires the `export` extra; optional backend cases additionally require `timm`, `transformers` and `bioclip`. CI's `all` extra includes these. Tests use randomly initialized offline models and check all classifier head families, representative backbones, state preservation, masks and structured outputs. @@ -137,15 +145,15 @@ skip cannot be observed reliably. Arbitrary custom internal no-ops are not infer by inspecting parameters. This boundary must be rechecked when PyTorch changes its fused AMP contract. -`tests/test_optimizer_steps.py` uses real GradScaler overflow, scale growth and +`tests/training/test_optimizer_steps.py` uses real GradScaler overflow, scale growth and recovery on CPU and optionally CUDA, including native fused AdamW/SGD. It checks parameters, optimizer state, scheduler state and EMA call indices, as well as zero-LR steps, scale underflow and hook cleanup. Checkpoint regressions additionally compare uninterrupted/resumed MuonAuxAdamW, AdamW and momentum SGD training. ```bash -bash dev/check.sh test tests/test_optimizer_steps.py tests/test_checkpoint_contract.py -RUN_CUDA_TESTS=1 CUDA_VISIBLE_DEVICES=0 bash dev/check.sh test tests/test_optimizer_steps.py -k cuda +bash dev/check.sh test tests/training/test_optimizer_steps.py tests/training/test_checkpoint_contract.py +RUN_CUDA_TESTS=1 CUDA_VISIBLE_DEVICES=0 bash dev/check.sh test tests/training/test_optimizer_steps.py -k cuda ``` The configured GPU benchmark workflow runs these CUDA regressions too. An explicitly @@ -154,3 +162,199 @@ hardware cases. These are optimizer/AMP tests, not an EMA-functionality claim. References: [optimizer post-step hooks](https://docs.pytorch.org/docs/2.12/generated/torch.optim.Optimizer.register_step_post_hook.html) and [GradScaler](https://docs.pytorch.org/docs/2.12/amp.html). + +### CUDA batch transfer lookahead + +`mt_train --cuda-prefetch` and `mt_predict --cuda-prefetch` opt into one-batch +transfer lookahead. Python loader builders accept `cuda_prefetch=True` with a CUDA +`device`. The returned object still inherits `DataLoader`, with the same sampler, +length, worker settings and repeated-epoch behavior; its batches are already on +the requested CUDA device. Shape, dtype and order are preserved. Model +preprocessing/augmentation stays on the caller's compute stream, so this works +independently of float32, AMP or INT8 model execution. + +The option defaults off, rejects CPU targets, and is bypassed for an already +CUDA-cached dataset. It stages one additional batch and records stream usage so +the allocator cannot recycle batch storage before consumption completes. It does +not increase worker counts or add CPU reader threads. Custom CPU hooks may run a +batch earlier; stochastic hooks sharing global RNG state can therefore change +their interleaving with caller code. Returned batches may outlive iteration; +callers using them on another CUDA stream must establish their own stream handoff. +See [PyTorch stream semantics](https://docs.pytorch.org/docs/main/notes/cuda.html#cuda-streams). + +This is an opt-in throughput/memory tradeoff. Actual gains depend on the balance +between transfer and compute; compare peak allocation as well as wall time using +[the transfer probe](benchmarks/training.md#capacity-and-bottleneck-probes). + +### Direct pinned cache batches + +For a CUDA target with `cache="CPU"` and `num_workers=0`, the shared training +loader now gathers cached rows directly into pinned batch storage. This removes +the intermediate pageable batch and its second copy during pinning. Batches own +their storage: modifying one cannot change the cache, and keeping an older batch +cannot cause it to be overwritten by a later iteration. Sampling, label order, +shape and dtype are unchanged. + +This applies automatically with either ordinary transfer or `--cuda-prefetch`. +Raw `LazyDataset` users can request `pin_batches=True` for the same behavior. +Worker processes always use the ordinary gather path and DataLoader's parent-side +pinning; this option never initializes the CUDA pin allocator in a worker. CPU +training, CUDA-cached datasets and scalar indexing keep their existing behavior. + +### Direct collation of stacked batches + +Repository loaders now retain the gathered batch tensors through collation, +avoiding creation of one image/label view per sample. Their batch sampler tags +index lists for this internal path; dataset identity, shuffle/drop-last behavior, +distributed sampler access and RNG consumption are preserved. + +Ordinary external `LazyDataset.__getitems__` calls still return actual sample +lists, including direct `torch.stack` compatibility. An external DataLoader that +reuses the repository batch sampler with its default collator materializes sample +views on demand. Tests cover image-only and image/label batches with both direct +loading and CPU caches, including spawned workers and CUDA prefetch. + +A one-thread cache benchmark with 4,096 uint8 RGB 28×28 images, batch size 128, +and seven alternating trials measured approximately 0.52 million samples/s before +this change and 2.22 million afterward. This isolates cached iteration; larger +images, decoding, transfer and model compute change the overall benefit. See the +[integrated measurements](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md#larger-batches-and-direct-collation). + +### Model compilation + +`mt_train --compile --compile-mode reduce-overhead` selects a PyTorch model +compilation mode. The Python training entry points accept `compile_mode`, and +`dev.benchmarks.training.run` accepts the same CLI flag and records it in success and +failure reports. An explicit mode requires `--compile`; omitting it preserves +ordinary `torch.compile(model)` behavior. Optimizer compilation remains separate. + +Supported modes are `default`, `reduce-overhead`, `max-autotune`, and +`max-autotune-no-cudagraphs`. PyTorch's CUDA graph modes can reduce launch overhead +for eligible graphs, but capture is not guaranteed and workspace caching can +increase memory. Measure both float and INT8 with the same mode, including +compilation time, later training phases, peak allocation and held-out quality. +See the [PyTorch compilation modes](https://docs.pytorch.org/docs/2.12/generated/torch.compile.html). + +### Optimizer compilation + +`mt_train --compile-optimizer` opts into compiling optimizer updates independently +of model `--compile`. The benchmark runner accepts the same option and records it +in result and failure reports. It defaults off; compilation overhead and graph +breaks can outweigh any steady-state benefit, so measure the intended workload. + +The first real optimizer call initializes lazy state eagerly. Subsequent calls +use compilation with tensor learning rates, allowing scheduler changes without +specializing a graph for every numeric rate. AMP overflow handling and scheduler +advancement remain controlled by the trainer. MuonAuxAdamW compiles its child +optimizers while keeping its outer step counter in Python; Muon's compilation +preserves the explicit BF16 casts in its Newton-Schulz iterations. + +For custom training, call `mini_trainer.training.compilation.compile_optimizer` +after constructing the scheduler and restoring checkpoint state. Saved learning +rates remain ordinary scalars, so a checkpoint can resume without compilation. +Explicit `foreach=True` Adam/AdamW requires `capturable=True`; unsupported +combinations fail before the helper changes the optimizer. Default and explicit +`foreach=False` updates do not need that setting. + +Regression coverage compares eager and compiled SGD, AdamW, their native fused +variants, and MuonAuxAdamW on CUDA, including overflow skips, scheduler changes, +parameter updates and optimizer state. This does not establish support for every +custom optimizer or a real-workload INT8 speedup. Quantized update dispatch and +its performance remain a separate validation boundary. + +### Optimizer CUDA graphs + +`mt_train --compile-optimizer --optimizer-cudagraphs` additionally requests CUDA +graph replay for optimizer updates. It requires parameters on one CUDA device +and the default Inductor backend. It is independent of model compilation; +`--compile --compile-mode reduce-overhead` can enable model graphs as well. +The benchmark runner accepts and records the same optimizer option. + +For custom training, call `compile_optimizer(optimizer, cudagraphs=True)` after +scheduler construction and checkpoint restoration. The first real update still +initializes state eagerly, under the trainer's AMP gating. Subsequent updates use +device-resident learning rates. Scheduler updates, overflow decisions and Muon's +outer step counter remain outside graph capture. Changing the graph setting of +an already compiled optimizer is rejected rather than silently ignored. + +Numeric rates retain float64 precision; explicitly supplied tensor rates retain +their dtype. Checkpoints save numeric values plus a `_mini_trainer_lr_dtype` +marker for non-default precision, allowing compiled restoration to recover that +choice. Ordinary eager loading still accepts the numeric rates. Native fused +SGD/Adam/AdamW tensor-rate kernels require an explicitly selected float32 tensor +rate (or its recorded checkpoint marker); unsupported rates fail before mutation. +The existing foreach/capturable restrictions still apply. + +INT8 updates use compiler-visible arithmetic and final storage copies during AOT +fake-tensor tracing. This avoids an opaque in-place operator carrying CPU scalar +inputs into CUDA graph partitions. The native row update remains available in +eager execution. Regression tests check actual optimizer-only replay, arithmetic, +AMP skips, rate precision, same-optimizer restoration and eager checkpoint resume. +Capture eligibility, extra gradient copies, graph workspace memory and first-use +compilation costs still depend on the optimizer and workload. Measure both float +and INT8 with the same options; enabling graphs alone is not evidence of a speedup. + +## Agent-only changes and CI + +Material primarily for coding agents lives in [`.agents/`](../.agents/README.md), +with root [`AGENTS.md`](../AGENTS.md) as the entry point. Durable notes use the +[shared format](../.agents/notes/README.md); scratch work belongs in ignored +`.agents/local/`. Developer-facing documentation remains in `docs/` and `dev/`. + +Use separate `agent:` commits for agent instructions and notes. This prefix marks +purpose, not authorship, and does not disable checks. CI workflow changes, executable +helpers and application changes receive normal separate commits. + +The CI and dataset benchmark workflows skip pushes that change only `AGENTS.md`, +Markdown inside `.agents/`, or `.agents/.gitignore`. Other Markdown, code, scripts, +configuration and workflow changes still run the usual checks. On pull requests, +a small `scope` job compares the complete PR diff; agent-only changes skip costly +jobs while preserving job statuses. Mixed changes run checks regardless of commit +messages. Scheduled and manual benchmarks keep their existing behavior. + +`dev/ci_scope.py` uses Git and the Python standard library, with no environment +installation. Missing/unreadable/empty comparisons run checks conservatively; +renames inspect both old and new paths. If classification fails, downstream checks +still run unless the workflow was cancelled. The push patterns are also checked +against the classifier by `tests/core/test_ci_scope.py`. + +Do not use `[skip ci]` as an alternative: GitHub documents that workflow-level +skipping can leave [required PR checks pending](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#onpushpull_requestpull_request_targetpathspaths-ignore). +Agent-only edits need content/link review and `git diff --check`; changes to the +classifier or workflows need their focused checks. Release-tag, scheduled and +manual workflows are not disabled by agent commit messages. + +## Automatic CPU budgets + +Automatic loader and cache worker counts now use the smallest detected process +CPU count, affinity mask, visible Linux cgroup CPU quota, and positive +`SLURM_CPUS_PER_TASK` allocation. Cgroup v1 and v2 ancestor limits are included; +fractional CPU quotas are rounded down before applying the existing four-CPU +reserve and worker caps. Explicit worker counts, including zero, remain unchanged. +Unreadable, unlimited, or malformed quota data falls back to the other signals. + +These are resource ceilings, not a measurement of contention from other jobs. +For a deliberately shared allocation, set worker counts explicitly when needed. +The relevant interfaces are documented by the +[Linux kernel](https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files) +and [Slurm](https://slurm.schedmd.com/sbatch.html#OPT_SLURM_CPUS_PER_TASK). + +CI runs on pull requests targeting `master` and on pushes to `master`. Feature +branches such as `quant` use PR checks, avoiding duplicate push/PR jobs. New +commits cancel superseded runs for the same PR or branch. The x86 quantization +job forces AVX2 to verify portability without relying on runner VNNI support. + +## PR change statistics + +`pr-change-summary.yml` maintains one bot comment per PR, with file and line counts +for the core module, tests, benchmark tooling, CI, packaging, and other areas. +Markdown is excluded from the headline except root `README.md`, whose onboarding +instructions are part of the user interface. Other Markdown remains visible in a +separate row. The workflow's `featureMarkdown` set is the explicit exception list; +add a path there when its content is itself a delivered feature. + +The workflow reads GitHub PR metadata only. It uses `pull_request_target` with +permission to comment, performs no checkout and never executes PR content. It +starts working once installed on the PR base branch. Counts use GitHub's PR diff, +classify renames by destination, and flag incomplete results above the API's +3,000-file limit. They measure change volume, not quality or development effort. diff --git a/dev/benchmarks/README.md b/dev/benchmarks/README.md index 66b4625f..99d1b2d5 100644 --- a/dev/benchmarks/README.md +++ b/dev/benchmarks/README.md @@ -18,7 +18,7 @@ bash dev/check-benchmarks.sh gpu /tmp/benchmarks-gpu BENCHMARK_DATA_ROOT="$PWD/examples" \ BLAIR_CLASS_SPEC="$PWD/examples/blair/blair_model/class_spec.json" \ bash dev/check-benchmarks.sh real /tmp/benchmarks-real -bash dev/check.sh test tests/test_benchmark_synthetic.py tests/test_benchmark_datasets.py +bash dev/check.sh test tests/benchmarks/test_benchmark_synthetic.py tests/benchmarks/test_benchmark_datasets.py ``` Each results directory must be new. Set `BENCHMARK_PYTHON` to use a different prepared @@ -32,9 +32,9 @@ specification; it performs no downloads or online taxonomy queries. Individual runs are configurable: ```bash -.venv/bin/python -m dev.benchmarks.run --output /tmp/oracle \ +.venv/bin/python -m dev.benchmarks.training.run --output /tmp/oracle \ --seed 42 --threads 1 --device cpu -.venv/bin/python -m dev.benchmarks.run --dataset mnist \ +.venv/bin/python -m dev.benchmarks.training.run --dataset mnist \ --data-root examples/mnist --output /tmp/mnist-amp --epochs 5 \ --device cuda:0 --dtype float16 --cache CUDA --allow-nondeterministic ``` @@ -106,8 +106,9 @@ module coverage from a passing profile. and weekly. It uses a locked CPU environment and uploads results even on failure. The Actions run page contains a readable summary and artifacts with 90-day retention. The README links to that history. The source checkout is not modified to publish results. -These are initial reporting facilities; durable archival and a historical dashboard -remain follow-up work. +These dataset artifacts still expire. A separate opt-in TensorRT publisher connects +compact records to draft storage and Pages; its [activation and live-verification +steps](reporting.md#opt-in-public-history-publisher) remain pending. GPU execution uses a configured self-hosted runner labeled `self-hosted`, `linux`, `gpu`. It creates a disposable uv environment with an explicitly selected CUDA extra, @@ -119,8 +120,42 @@ For real-data jobs, enable input `real_data` together with `gpu`, or set `BENCHMARK_REAL_DATA=true`. Set repository variables `BENCHMARK_DATA_ROOT` and `BLAIR_CLASS_SPEC` to stable dataset and specification paths on that runner, outside the checkout. The workflow does not assume these resources already exist and cannot -validate GPU behavior on a CPU-only hosted runner. No GPU job has been dispatched -from this development session; the equivalent local commands have been exercised. +validate GPU behavior on a CPU-only hosted runner. Local command validation does +not establish that the hosted GPU workflow has run successfully; retain the actual +workflow run and target-machine evidence when qualifying it. References: [Actions job summaries](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#adding-a-job-summary) and [artifact retention](https://github.com/actions/upload-artifact#retention-period). + +## Focused guides + +| Task | Guide | +| --- | --- | +| Native QT, representative training, capacity, loading, convergence and training predictions | [Training](training.md) | +| Input preparation, calibration, ONNX CPU/TensorRT builds, placement, quality and isolated resources | [Inference](inference.md) | +| Target workflow, compact CPU/GPU history, draft storage and Pages activation | [Reporting](reporting.md) | +| Current findings and their limits | [Benchmark findings](../../docs/benchmarks.md) | +| Ordered branch work, bottlenecks and hardware dependencies | [Quantization roadmap](../../docs/quantization-roadmap.md) | + +Use the shared commands in these guides; they do not implicitly synchronize the +working environment. Historical experiments and rejected approaches are retained +in the [experiment archive](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md). + +## Source layout and command migration + +| Package | Ownership | +| --- | --- | +| `training/` | Dataset training, native QT/PTQ probes, large heads, prediction adapter | +| `data/` | Dataset generation/indexing, loading, caching, reading and transfer probes | +| `inference/` | Input preparation, calibration, ONNX/TensorRT, paired quality/resources | +| `reporting/` | Summaries, immutable history, release storage | + +Development CLI modules now include their package: for example, +`dev.benchmarks.run` becomes `dev.benchmarks.training.run` and +`dev.benchmarks.cpu_deployment` becomes `dev.benchmarks.inference.cpu_deployment`. +The shared `dev/check-*.sh` entry points are unchanged. `models.py` stays at its +original import path because saved benchmark checkpoints name those classes. +`prepare_inputs.py` retains the old preprocessing-factory import for saved recipes; +`_int8_weight.py` retains the earlier probe compatibility import. +Source hashes change with this reorganization; historical reports keep their +original hashes and should not be relabeled as runs of the new revision. diff --git a/dev/benchmarks/_int8_weight.py b/dev/benchmarks/_int8_weight.py new file mode 100644 index 00000000..161cbf94 --- /dev/null +++ b/dev/benchmarks/_int8_weight.py @@ -0,0 +1,5 @@ +"""Compatibility import for earlier developer probes.""" + +from mini_trainer.modeling._quantized_training import TrainingWeight + +__all__ = ["TrainingWeight"] diff --git a/dev/benchmarks/data/__init__.py b/dev/benchmarks/data/__init__.py new file mode 100644 index 00000000..99c8fd2c --- /dev/null +++ b/dev/benchmarks/data/__init__.py @@ -0,0 +1 @@ +"""Development benchmarks: data.""" diff --git a/dev/benchmarks/data/cache.py b/dev/benchmarks/data/cache.py new file mode 100644 index 00000000..bb893da6 --- /dev/null +++ b/dev/benchmarks/data/cache.py @@ -0,0 +1,95 @@ +"""Measure CPU cache construction from reproducible PNGs; exclude file creation. + +An optional developer-supplied baseline io.py is executed as Python source to +compare an earlier implementation in the same dependency environment. +""" + +import gc +import hashlib +import importlib.util +import json +import statistics +import sys +import tempfile +import time +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from mini_trainer.data._workers import _available_cpu_count +from mini_trainer.data.io import LazyDataset, make_read_and_resize_fn +from mini_trainer.data.loader import PathLabelProcessor + + +def run(samples=2048, size=128, repeats=5, workers=None, baseline_source=None): + implementations = {"bounded": LazyDataset} + if baseline_source is not None: + name = "mini_trainer.data._benchmark_baseline" + spec = importlib.util.spec_from_file_location(name, baseline_source) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + implementations["baseline"] = module.LazyDataset + reader = PathLabelProcessor(make_read_and_resize_fn((size, size), torch.device("cpu"), torch.uint8), None, False) + timings = {name: [] for name in implementations} + generator = np.random.default_rng(42) + with tempfile.TemporaryDirectory(prefix="mini-trainer-cache-probe-") as root: + paths = [] + # Repeat a fixed small corpus, measuring warm filesystem-cache decoding. + for index in range(min(64, samples)): + path = Path(root) / f"{index}.png" + Image.fromarray(generator.integers(0, 256, (size, size, 3), dtype=np.uint8)).save(path) + paths.append(str(path)) + items = ([paths[index % len(paths)] for index in range(samples)], list(range(samples))) + expected_first = reader((items[0][0], 0))[0] + for trial in range(repeats + 1): + order = list(implementations) if trial % 2 else list(reversed(implementations)) + for name in order: + gc.collect() + options = {"cache_workers": workers} if name == "bounded" else {} + started = time.perf_counter() + dataset = implementations[name](reader, items, cache="cpu", **options) + seconds = time.perf_counter() - started + assert len(dataset) == samples + torch.testing.assert_close(dataset[0][0], expected_first, rtol=0, atol=0) + assert torch.equal(dataset._ram_cache.tensors[1], torch.arange(samples)) + if trial: + timings[name].append(seconds) + del dataset + return { + "seed": 42, + "samples": samples, + "shape": [3, size, size], + "threads": torch.get_num_threads(), + "cache_workers": workers, + "available_cpus": _available_cpu_count(), + "source_sha256": { + name: hashlib.sha256(Path(sys.modules[implementation.__module__].__file__).read_bytes()).hexdigest() + for name, implementation in implementations.items() + }, + "scope": "CPU cache construction from 64 repeated PNGs; warm filesystem cache; excludes file generation", + "seconds": timings, + "median_samples_per_second": {name: samples / statistics.median(values) for name, values in timings.items()}, + "speedup": statistics.median(timings["baseline"]) / statistics.median(timings["bounded"]) if baseline_source else None, + } + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--samples", type=int, default=2048) + parser.add_argument("--size", type=int, default=128) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--workers", type=int) + parser.add_argument("--baseline-source", type=Path) + args = parser.parse_args() + if min(args.samples, args.size, args.repeats) < 1 or (args.workers is not None and args.workers < 0): + parser.error("Sizes and repeats must be positive; workers must be nonnegative") + torch.set_num_threads(1) + print(json.dumps(run(**vars(args)), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/datasets.py b/dev/benchmarks/data/datasets.py similarity index 100% rename from dev/benchmarks/datasets.py rename to dev/benchmarks/data/datasets.py diff --git a/dev/benchmarks/data/loader.py b/dev/benchmarks/data/loader.py new file mode 100644 index 00000000..9e46ac6f --- /dev/null +++ b/dev/benchmarks/data/loader.py @@ -0,0 +1,124 @@ +"""Measure scalar versus batched loading without changing data or sampling.""" + +import json +import statistics +import time +from argparse import ArgumentParser + +import torch +from torch.utils.data import DataLoader, Dataset + +from mini_trainer.data.io import LazyDataset +from mini_trainer.data.loader import get_dataloader + + +class ScalarFetch(Dataset): + """The previous per-sample fetch contract, deliberately without __getitems__.""" + + def __init__(self, dataset): + self.dataset = dataset + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, index): + return self.dataset[index] + + +class TensorReader: + """Picklable synthetic reader for spawn-worker batch assembly probes.""" + + def __init__(self, images): + self.images = images + + def __call__(self, item): + return self.images[item[0]], torch.tensor(item[0]) + + +def run(samples=2048, size=64, batch_size=64, repeats=5, pin_batches=False, workers=0, cache="cpu"): + if pin_batches and (workers or cache != "cpu"): + raise ValueError("Pinned gathering comparison requires CPU caching and zero workers.") + generator = torch.Generator().manual_seed(42) + images = torch.randint(0, 256, (samples, 3, size, size), dtype=torch.uint8, generator=generator) + reader = TensorReader(images) + dataset = LazyDataset(reader, (list(range(samples)),), cache=cache, cache_workers=0) + context = "spawn" if workers else None + loaders = { + "scalar": DataLoader( + ScalarFetch(dataset), + batch_size=batch_size, + num_workers=workers, + persistent_workers=workers > 0, + multiprocessing_context=context, + ), + "batched": get_dataloader(dataset, "val", batch_size, workers, False, torch.device("cpu"), multiprocessing_context=context), + } + if pin_batches: + if not torch.cuda.is_available(): + raise RuntimeError("Pinned batch comparison requires an accessible CUDA device.") + direct = LazyDataset( + reader, + (list(range(samples)),), + cache="cpu", + cache_workers=0, + pin_batches=True, + ) + loaders = { + "gather_then_pin": get_dataloader(dataset, "val", batch_size, 0, True, torch.device("cuda:0")), + "pinned_gather": get_dataloader(direct, "val", batch_size, 0, True, torch.device("cuda:0")), + } + baseline, candidate = list(loaders) + for expected, actual in zip(loaders[baseline], loaders[candidate], strict=True): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + timings = {name: [] for name in loaders} + for trial in range(repeats + 1): + # Alternate order to avoid consistently giving one variant warm caches. + for name in list(loaders) if trial % 2 else list(reversed(loaders)): + started = time.perf_counter() + count = sum(len(batch[0]) for batch in loaders[name]) + elapsed = time.perf_counter() - started + assert count == samples + if trial: + timings[name].append(elapsed) + return { + "samples": samples, + "shape": [3, size, size], + "dtype": "uint8", + "batch_size": batch_size, + "workers": workers, + "cache": cache, + "threads": torch.get_num_threads(), + "identical_batches": True, + "pin_batches": pin_batches, + "scope": ( + "synthetic tensor loader iteration including worker IPC; " + "excludes worker startup, cache construction, image decoding, preprocessing, H2D and model compute" + ), + "seconds": timings, + "median_samples_per_second": {name: samples / statistics.median(values) for name, values in timings.items()}, + "speedup": statistics.median(timings[baseline]) / statistics.median(timings[candidate]), + } + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--samples", type=int, default=2048) + parser.add_argument("--size", type=int, default=64) + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--pin-batches", action="store_true") + parser.add_argument("--workers", type=int, default=0) + parser.add_argument("--cache", choices=["cpu", "none"], default="cpu") + args = parser.parse_args() + if min(args.samples, args.size, args.batch_size, args.repeats) < 1: + parser.error("All sizes and repeat counts must be positive") + if args.workers < 0: + parser.error("Worker count must be nonnegative") + if args.pin_batches and (args.workers or args.cache != "cpu"): + parser.error("--pin-batches requires --cache cpu and --workers 0") + torch.set_num_threads(1) + print(json.dumps(run(**vars(args)), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/data/reader.py b/dev/benchmarks/data/reader.py new file mode 100644 index 00000000..b847df75 --- /dev/null +++ b/dev/benchmarks/data/reader.py @@ -0,0 +1,94 @@ +"""Compare streaming image readers with exact batches and file provenance.""" + +import hashlib +import json +import statistics +import time +from argparse import ArgumentParser +from importlib.metadata import version +from pathlib import Path + +import torch +from torchvision.io import ImageReadMode, decode_image +from torchvision.transforms import functional as transforms + +from mini_trainer.data.io import ReadAndResize +from mini_trainer.data.loader import get_inference_dataloader + + +class TorchvisionReader(ReadAndResize): + """The former decode/float-resize/convert path, retained as a reference.""" + + def __call__(self, path): + if not isinstance(path, str): + path = path[0] + image = decode_image(path, mode=ImageReadMode.RGB, apply_exif_orientation=False) + image = transforms.resize(image, [self.h, self.w], interpolation=self.interp, antialias=self.antialias) + return (image if image.dtype == self.dtype else self.converter(image)).to(self.device) + + +def run(data_root, samples=128, size=224, batch_size=16, repeats=7, workers=0): + root = Path(data_root).resolve() + paths = sorted(path for path in root.rglob("*") if path.suffix.lower() in (".jpg", ".jpeg", ".png") and path.is_file())[:samples] + if not paths: + raise ValueError("No JPEG/PNG images found under the supplied data root.") + manifest = [{"path": str(path.relative_to(root)), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} for path in paths] + loaders = {} + for name in ("torchvision", "gather"): + dataset, loaders[name] = get_inference_dataloader( + list(map(str, paths)), + resize_size=size, + batch_size=batch_size, + num_workers=workers, + multiprocessing_context="spawn" if workers else None, + ) + if name == "torchvision": + dataset.func = TorchvisionReader((size, size), torch.device("cpu"), torch.uint8) + for expected, actual in zip(loaders["torchvision"], loaders["gather"], strict=True): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + timings = {name: [] for name in loaders} + for trial in range(repeats + 1): + for name in list(loaders) if trial % 2 else list(reversed(loaders)): + started = time.perf_counter() + count = sum(len(batch) for batch in loaders[name]) + elapsed = time.perf_counter() - started + assert count == len(paths) + if trial: + timings[name].append(elapsed) + return { + "files": manifest, + "samples": len(paths), + "shape": [3, size, size], + "dtype": "uint8", + "batch_size": batch_size, + "workers": workers, + "threads": torch.get_num_threads(), + "versions": {name: version(name) for name in ("torch", "torchvision", "numpy")}, + "identical_batches": True, + "scope": ( + "uncached image loading, decoding, nearest resize, batch assembly and IPC with a warm filesystem cache; " + "excludes worker startup, H2D and model compute" + ), + "seconds": timings, + "median_samples_per_second": {name: len(paths) / statistics.median(values) for name, values in timings.items()}, + "speedup": statistics.median(timings["torchvision"]) / statistics.median(timings["gather"]), + } + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--data-root", type=Path, required=True) + parser.add_argument("--samples", type=int, default=128) + parser.add_argument("--size", type=int, default=224) + parser.add_argument("--batch-size", type=int, default=16) + parser.add_argument("--repeats", type=int, default=7) + parser.add_argument("--workers", type=int, default=0) + args = parser.parse_args() + if min(args.samples, args.size, args.batch_size, args.repeats) < 1 or args.workers < 0: + parser.error("Sizes/repeats must be positive and workers nonnegative.") + torch.set_num_threads(1) + print(json.dumps(run(**vars(args)), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/synthetic.py b/dev/benchmarks/data/synthetic.py similarity index 100% rename from dev/benchmarks/synthetic.py rename to dev/benchmarks/data/synthetic.py diff --git a/dev/benchmarks/data/transfer.py b/dev/benchmarks/data/transfer.py new file mode 100644 index 00000000..bedaa33e --- /dev/null +++ b/dev/benchmarks/data/transfer.py @@ -0,0 +1,108 @@ +"""Compare same-stream H2D with one-batch lookahead around fixed ResNet compute.""" + +import json +import statistics +import time +from argparse import ArgumentParser + +import torch +from torchvision.models import resnet18 + +from mini_trainer.data.io import LazyDataset +from mini_trainer.data.loader import get_dataloader + + +def run(samples=512, size=224, batch_size=32, repeats=5, backward=False, dtype="float16"): + if not torch.cuda.is_available(): + raise RuntimeError("CUDA transfer benchmark requires an accessible CUDA device.") + torch.manual_seed(42) + device = torch.device("cuda:0") + images = torch.randint(0, 256, (samples, 3, size, size), dtype=torch.uint8) + dataset = LazyDataset(lambda item: images[item[0]], (list(range(samples)),), cache="cpu", cache_workers=0) + direct = LazyDataset( + lambda item: images[item[0]], + (list(range(samples)),), + cache="cpu", + cache_workers=0, + pin_batches=True, + ) + variants = { + "same_stream": (dataset, False), + "prefetch": (dataset, True), + "pinned_gather": (direct, False), + "pinned_gather_prefetch": (direct, True), + } + loaders = { + name: get_dataloader(data, "val", batch_size, 0, True, device, cuda_prefetch=prefetch) + for name, (data, prefetch) in variants.items() + } + model = resnet18(weights=None).eval().to(device) + precision = getattr(torch, dtype) + timings = {name: [] for name in loaders} + peaks = {name: [] for name in loaders} + reference = None + for trial in range(repeats + 1): + for name in list(loaders) if trial % 2 else list(reversed(loaders)): + model.zero_grad(set_to_none=True) + torch.cuda.synchronize(device) + torch.cuda.reset_peak_memory_stats(device) + started = time.perf_counter() + outputs = [] + with torch.set_grad_enabled(backward): + for batch in loaders[name]: + inputs = batch.to(device, non_blocking=True).float().div_(255) + with torch.autocast("cuda", dtype=precision, enabled=precision != torch.float32): + output = model(inputs) + outputs.append(output.detach()) + if backward: + output.float().square().mean().backward() + model.zero_grad(set_to_none=True) + torch.cuda.synchronize(device) + elapsed = time.perf_counter() - started + peak = torch.cuda.max_memory_allocated(device) + actual = torch.cat(outputs).cpu() + if reference is None: + reference = actual + torch.testing.assert_close(actual, reference, rtol=0, atol=0) + if trial: + timings[name].append(elapsed) + peaks[name].append(peak) + return { + "samples": samples, + "size": size, + "batch_size": batch_size, + "workers": 0, + "dtype": dtype, + "backward": backward, + "device": torch.cuda.get_device_name(device), + "torch_version": torch.__version__, + "identical_outputs": True, + "scope": ( + "CPU cache iteration, pinned H2D, scaling and fixed ResNet18 compute; " + "BN frozen; no optimizer updates; excludes cache/model construction" + ), + "seconds": timings, + "peak_cuda_allocated_bytes": peaks, + "median_samples_per_second": {name: samples / statistics.median(values) for name, values in timings.items()}, + "speedup": statistics.median(timings["same_stream"]) / statistics.median(timings["prefetch"]), + "speedups": {name: statistics.median(timings["same_stream"]) / statistics.median(values) for name, values in timings.items()}, + } + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--samples", type=int, default=512) + parser.add_argument("--size", type=int, default=224) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--backward", action="store_true") + parser.add_argument("--dtype", choices=["float32", "float16", "bfloat16"], default="float16") + args = parser.parse_args() + if min(args.samples, args.size, args.batch_size, args.repeats) < 1: + parser.error("Sizes and repeats must be positive") + torch.set_num_threads(1) + print(json.dumps(run(**vars(args)), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference.md b/dev/benchmarks/inference.md new file mode 100644 index 00000000..a86b1016 --- /dev/null +++ b/dev/benchmarks/inference.md @@ -0,0 +1,126 @@ +# Inference benchmark workflow + +All commands run from the checkout in explicitly prepared environments; they do +not install dependencies. Use fresh output directories and retain failed reports. +Use `python -m dev.benchmarks.inference.MODULE --help` for all options. + +## Maintained image input preparation + +Prepare calibration from training data and evaluation from the complete held-out +split, using the benchmark dataset manifest and exported model's preprocessing: + +```bash +.venv/bin/python -m dev.benchmarks.inference.prepare_inputs \ + --dataset-manifest benchmark/dataset_manifest.json --data-root examples/blair \ + --export-manifest exported/manifest.json --output prepared-train \ + --split train --count 128 --seed 42 --batch-size 8 --score-semantics logits +.venv/bin/python -m dev.benchmarks.inference.prepare_inputs \ + --dataset-manifest benchmark/dataset_manifest.json --data-root examples/blair \ + --export-manifest exported/manifest.json --output prepared-val \ + --split val --batch-size 8 --score-semantics logits +``` + +Class order, source image hashes, input shape/dtype and selected identities are +checked and retained. Declare actual score semantics; arbitrary model outputs +are not necessarily logits. Omit `--count` for the complete split in manifest order. +Custom preprocessing needs an importable factory; the default uses repository +preprocessing on CPU with zero workers. Preparation checks declared split hashes +but cannot establish that an externally supplied split policy is appropriate. + +## Maintained ONNX calibration command + +```bash +.venv/bin/python -m dev.benchmarks.inference.onnx_calibration \ + --model exported/model.onnx --manifest prepared-train/manifest.json \ + --output qdq-cpu --threads 1 +.venv/bin/python -m dev.benchmarks.inference.onnx_calibration \ + --model exported/model.onnx --manifest prepared-train/manifest.json \ + --output qdq-trt --threads 1 --activation-type int8 --symmetric-activations --float-bias +``` + +The CPU recipe defaults to unsigned activations, symmetric per-channel INT8 +weights, quantized biases and Percentile 99.9 calibration over Conv/Gemm/MatMul. +The TensorRT candidate uses signed symmetric activation quantizers and floating +biases. Histogram symmetry is separate; the tested recipe does not select +`--symmetric-calibration`. Neither recipe is automatically qualified for a target. + +Retain ordered NPZ batches/sample IDs, preprocessing, calibration provenance, +source/external-weight hashes, ranges and recipe report. Calibration inputs must +be finite and match all named model inputs. A calibration smoke run is not +held-out quality or proof of integer execution. + +Native QT checkpoints can be explicitly materialized to floating deployment +weights before static calibration; see [export](../../docs/onnx.md). That route +changes the dynamic activation-quantization contract and is not a resume conversion. + +## Composed CPU deployment comparison + +```bash +.venv/bin/python -m dev.benchmarks.inference.cpu_deployment \ + --baseline exported/model.onnx --candidate qdq-cpu/model.onnx \ + --manifest prepared-val/manifest.json --inputs batch-one.npz \ + --output cpu-comparison --threads 1 --trials 3 --warmup 3 --repeats 31 \ + --require-provider-op QLinearConv --require-provider-op QGemm \ + --metrics-python /path/to/metrics-env/bin/python +``` + +The manifest supplies held-out identities, class ordering and score semantics; +the separate NPZ supplies the representative resource-measurement shape. Use the +same preprocessed arrays for both models and retain their selection provenance. +Operator requirements are explicit examples; select the actual required operators +for your candidate rather than assuming every architecture uses QGemm. + +Quality, placement and each resource trial run separately. Trial order alternates, +and hashes/settings/runtime identities must agree. CPU ratios compare separate- +process warm latency medians, process RSS and approximate peak RSS. Placement +counts show remaining floating operations; requiring one integer operator does +not prove complete quantization. Requested trial/warmup/repeat budgets are retained. + +## Composed TensorRT deployment evaluation + +Prefer [the target harness](reporting.md#opt-in-target-gpu-workflow), which rebuilds +both engines on the target and evaluates them. For separately prepared engines: + +```bash +CUDA_VISIBLE_DEVICES=0 OMP_NUM_THREADS=1 python -m dev.benchmarks.inference.tensorrt_deployment \ + --baseline-build results/fp16 --candidate-build results/int8 \ + --manifest prepared-val/manifest.json --inputs prepared-val/batch-00000.npz \ + --output trt-comparison --metrics-python /path/to/metrics-env/bin/python \ + --trials 3 --warmup 10 --repeats 31 --memory-runs 20 --threads 1 +``` + +Build directories must contain matching `model.engine`, `report.json` and +`layers.json`, created by `inference.tensorrt_build`. Use matching profiles/settings +and a practical FP16 baseline. Engines are target-specific; keep ONNX external +weights beside source models. All resource shapes must fit both profiles. + +Evaluation checks build/engine/input identities, then collects held-out quality, +adjacent paired GPU latency and isolated single-engine memory trials. Both engines +coexist during timing. Memory snapshots come from separate processes and are +not transient peaks or per-process device allocation. + +## Individual tools + +| Module in `dev.benchmarks.inference` | Purpose | +| --- | --- | +| `onnx_inference` | Warm timing, output parity and provider/operator placement | +| `onnx_cpu_memory` | Fresh-process CPU latency, RSS and approximate peak | +| `tensorrt_build` | Build engine from ONNX with explicit profiles; retain layer inspection | +| `tensorrt_pair` | Adjacent paired engine timing | +| `tensorrt_memory` | Isolated engine memory stages | +| `dataset_inference` | Full held-out predictions and canonical CSVs | +| `inference_pair` | Compose baseline/candidate collection and quality | +| `quality_compare` | Five-metric comparison through prepared `mini_metrics` | + +## Interpretation and retention + +Read Macro-F1, Macro-Recall, Macro-Precision, Coverage and Theil's U at every +classification level. Preserve undefined metrics and inspect quality together with +resource benefit. `evaluated` means execution completed, not production acceptance. + +Retain source bundles, manifests/NPZs and complete output directories: child hashes, +commands/logs, predictions, metrics, raw timings, failures and `summary.md`. +[Reporting](reporting.md) projects compact visible history without replacing these +reproduction inputs. Host timings exclude preprocessing/loading; qualify that cost +separately on idle target hardware. CPU tests and x86 measurements do not establish +ARM execution; provider fallback is not integer GPU inference. diff --git a/dev/benchmarks/inference/__init__.py b/dev/benchmarks/inference/__init__.py new file mode 100644 index 00000000..95f8694f --- /dev/null +++ b/dev/benchmarks/inference/__init__.py @@ -0,0 +1 @@ +"""Development benchmarks: inference.""" diff --git a/dev/benchmarks/inference/cpu_deployment.py b/dev/benchmarks/inference/cpu_deployment.py new file mode 100644 index 00000000..e8bc8ed5 --- /dev/null +++ b/dev/benchmarks/inference/cpu_deployment.py @@ -0,0 +1,176 @@ +"""Compose CPU deployment quality, placement and isolated resource measurements.""" + +import json +import os +import subprocess +import sys +from argparse import ArgumentParser +from pathlib import Path + +from .inference_pair import run_pair +from .onnx_inference import file_hash + + +def evaluate( + baseline, candidate, manifest, inputs, output, threads=1, trials=3, warmup=3, repeats=31, required_ops=(), metrics_python=sys.executable +): + if min(threads, trials, warmup, repeats) < 1: + raise ValueError("Threads, trials, warmup and repeats must be positive") + paths = {name: Path(path).resolve(strict=True) for name, path in (("baseline", baseline), ("candidate", candidate))} + manifest, inputs, output = Path(manifest).resolve(strict=True), Path(inputs).resolve(strict=True), Path(output).resolve() + inputs_hash = file_hash(inputs) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "settings": {"threads": threads, "trials": trials, "warmup": warmup, "repeats": repeats}, + "required_candidate_ops": list(required_ops), + "stages": [], + "pairs": [], + "scope": ( + "CPU quality, observed operation placement and separate-process resource trials; no automatic acceptance gate. " + "Latency excludes image decoding/preprocessing. Peak RSS is approximate and includes process overhead. " + "Target claims require execution on that target; no thermal control or disk-cache eviction." + ), + } + + def save(): + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + + def child(name, module, arguments, expected): + command = [sys.executable, "-m", f"dev.benchmarks.inference.{module}", *map(str, arguments), "--output", str(output / name)] + stage = {"name": name, "command": command, "status": "running", "log": f"{name}.log"} + report["stages"].append(stage) + save() + with (output / stage["log"]).open("w") as log: + process = subprocess.run( + command, + cwd=Path(__file__).resolve().parents[3], + env={**os.environ, "PYTHONHASHSEED": "0"}, + stdout=log, + stderr=subprocess.STDOUT, + check=False, + ) + stage["returncode"] = process.returncode + if process.returncode: + raise RuntimeError(f"{name} failed; see {output / stage['log']}") + result_path = output / name / "report.json" + result = json.loads(result_path.read_text()) + if result["status"] != expected: + raise RuntimeError(f"Unexpected {name} status: {result['status']}") + stage.update(status=expected, report_sha256=file_hash(result_path)) + return result + + try: + report["phase"] = "quality" + save() + runtime = {"backend": "onnx", "threads": threads} + quality = run_pair(manifest, paths["baseline"], paths["candidate"], output / "quality", runtime, runtime, metrics_python) + report["quality"] = { + "report_sha256": file_hash(output / "quality/report.json"), + "levels": quality["levels"], + "models": quality["models"], + "undefined_metrics": quality["undefined_metrics"], + } + expected_files = {role: json.loads((output / f"quality/{role}/report.json").read_text())["model_files"] for role in paths} + report["phase"] = "placement" + report["execution"] = {} + for role, model in paths.items(): + args = [ + "--model", + model, + "--inputs", + inputs, + "--provider", + "CPUExecutionProvider", + "--threads", + threads, + "--warmup", + 1, + "--repeats", + 1, + ] + if role == "candidate": + for operation in required_ops: + args.extend(["--require-provider-op", operation]) + placement = child(f"placement-{role}", "onnx_inference", args, "passed") + if placement["models"][0]["files"] != expected_files[role] or placement["inputs"]["sha256"] != inputs_hash: + raise ValueError("Model or timing inputs changed between quality and placement") + report["execution"][role] = placement["models"][0]["execution"] + report["phase"] = "resources" + for trial in range(trials): + pair = {} + order = ("baseline", "candidate") if trial % 2 == 0 else ("candidate", "baseline") + for role in order: + result = child( + f"trial-{trial}-{role}", + "onnx_cpu_memory", + ["--model", paths[role], "--inputs", inputs, "--threads", threads, "--warmup", warmup, "--repeats", repeats], + "measured", + ) + if result["model_files"] != expected_files[role] or result["inputs"]["sha256"] != inputs_hash: + raise ValueError("Model or timing inputs changed between quality and resource measurement") + pair[role] = result + a, b = pair["baseline"], pair["candidate"] + if any(a[key] != b[key] for key in ("settings", "versions", "environment", "runtime_build", "memory_sources")): + raise ValueError("Paired resource environments or settings differ") + ratios = {"warm_latency": b["median_seconds"] / a["median_seconds"]} + for key in ("resident_bytes", "peak_resident_bytes"): + ratios[key] = b["memory"]["after_measurement"][key] / a["memory"]["after_measurement"][key] + report["pairs"].append({"trial": trial, "order": list(order), "candidate_over_baseline": ratios}) + save() + lines = [ + "# CPU deployment comparison", + "", + report["scope"], + "", + "| Trial | Warm latency ratio | Resident memory ratio | Approximate peak ratio |", + "| --- | ---: | ---: | ---: |", + ] + for pair in report["pairs"]: + ratios = pair["candidate_over_baseline"] + lines.append( + f"| {pair['trial']} | {ratios['warm_latency']:.4f} | {ratios['resident_bytes']:.4f} | {ratios['peak_resident_bytes']:.4f} |" + ) + lines.extend( + [ + "", + "Ratios are candidate / baseline; below one is lower. " + "These compare separate-process medians, not adjacent inference pairs.", + "", + (output / "quality/summary.md").read_text(), + ] + ) + (output / "summary.md").write_text("\n".join(lines)) + report.update(status="evaluated", phase="complete") + except BaseException as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + for stage in report["stages"]: + if stage["status"] == "running": + stage["status"] = "failed" + raise + finally: + save() + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + for name in ("baseline", "candidate", "manifest", "inputs", "output"): + parser.add_argument("--" + name, type=Path, required=True) + for name, default in (("threads", 1), ("trials", 3), ("warmup", 3), ("repeats", 31)): + parser.add_argument("--" + name, type=int, default=default) + parser.add_argument( + "--require-provider-op", + dest="required_ops", + action="append", + default=[], + help="Required candidate CPU operation type; inspect counts for coverage", + ) + parser.add_argument("--metrics-python", default=sys.executable) + evaluate(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/dataset_inference.py b/dev/benchmarks/inference/dataset_inference.py new file mode 100644 index 00000000..9ed5918c --- /dev/null +++ b/dev/benchmarks/inference/dataset_inference.py @@ -0,0 +1,276 @@ +"""Collect held-out ONNX/TensorRT predictions for the paired quality evaluator.""" + +import csv +import hashlib +import json +import platform +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np + +from .onnx_calibration import load_batch +from .onnx_inference import file_hash, model_files +from .quality_compare import COLUMNS, validate_dataset + + +def inference_manifest(path): + payload = Path(path).read_bytes() + manifest = json.loads(payload) + validate_dataset(manifest) + if not isinstance(manifest.get("batch_input"), str) or not manifest["batch_input"]: + raise ValueError("Declare batch_input for sample identity") + outputs = [] + for level in manifest["levels"]: + if ( + not isinstance(level.get("output"), str) + or not level["output"] + or level.get("score_semantics") not in ("logits", "probabilities") + ): + raise ValueError("Each level needs an output name and logits/probabilities score_semantics") + outputs.append(level["output"]) + if len(set(outputs)) != len(outputs): + raise ValueError("Output bindings must be unique") + batches = manifest.get("batches") + if not isinstance(batches, list) or not batches: + raise ValueError("Supply ordered input batches") + ids = [] + for batch in batches: + if not isinstance(batch.get("path"), str) or not batch["path"]: + raise ValueError("Each batch needs an NPZ path") + if not isinstance(batch.get("sample_ids"), list) or not batch["sample_ids"]: + raise ValueError("Each batch needs sample_ids as canonical integer strings") + ids.extend(batch["sample_ids"]) + if ( + any(not isinstance(i, str) for i in ids) + or len(set(ids)) != len(ids) + or set(ids) != {str(s["instance_id"]) for s in manifest["samples"]} + ): + raise ValueError("Batches must cover every declared sample exactly once using canonical string IDs") + return manifest, hashlib.sha256(payload).hexdigest() + + +def predictions(values, level, count): + values = np.asarray(values) + if values.shape != (count, len(level["classes"])) or values.dtype.kind != "f" or not np.isfinite(values).all(): + raise ValueError(f"Expected finite floating [batch,classes] scores for {level['name']}") + indices = values.argmax(axis=1) + if level["score_semantics"] == "probabilities": + if (values < 0).any() or (values > 1).any() or not np.allclose(values.sum(axis=1, dtype=np.float64), 1, rtol=1e-5, atol=1e-5): + raise ValueError("Declared probabilities must lie in [0,1] and sum to one") + confidence = values[np.arange(count), indices] + else: + shifted = values.astype(np.float64) - values.max(axis=1, keepdims=True) + confidence = 1 / np.exp(shifted).sum(axis=1) + return indices, confidence + + +class OnnxPredictor: + def __init__(self, model, report, provider, provider_options, threads, optimization): + import onnx + import onnxruntime as ort + + if provider not in ort.get_available_providers(): + raise ValueError(f"Requested provider unavailable: {provider}") + report["model_files"] = model_files(Path(model), onnx) + options = ort.SessionOptions() + options.intra_op_num_threads, options.inter_op_num_threads = threads, 1 + options.graph_optimization_level = getattr( + ort.GraphOptimizationLevel, f"ORT_{'DISABLE_ALL' if optimization == 'disable' else 'ENABLE_ALL'}" + ) + self.session = ort.InferenceSession(str(model), sess_options=options, providers=[provider], provider_options=[provider_options]) + self.session.disable_fallback() + if provider not in self.session.get_providers(): + raise RuntimeError("Requested provider was not activated") + self.names = [node.name for node in self.session.get_outputs()] + report["runtime"] = { + "onnx": onnx.__version__, + "onnxruntime": ort.__version__, + "providers": self.session.get_providers(), + "provider_options": self.session.get_provider_options(), + } + + def __call__(self, feeds): + return dict(zip(self.names, self.session.run(None, feeds), strict=True)) + + +class TensorRTPredictor: + def __init__(self, model, report, device): + import tensorrt as trt + import torch + + self.torch, self.trt, self.device = torch, trt, device + self.logger = trt.Logger(trt.Logger.WARNING) + trt.init_libnvinfer_plugins(self.logger, "") + self.runtime = trt.Runtime(self.logger) + payload = Path(model).read_bytes() + report["model_files"] = [{"path": str(Path(model).resolve()), "sha256": hashlib.sha256(payload).hexdigest(), "bytes": len(payload)}] + with torch.cuda.device(device): + self.engine = self.runtime.deserialize_cuda_engine(payload) + if self.engine is None: + raise RuntimeError("Could not deserialize TensorRT engine") + self.context = self.engine.create_execution_context() + if self.context is None: + raise RuntimeError("Could not create TensorRT execution context") + self.stream = torch.cuda.Stream(device=device) + report["runtime"] = { + "tensorrt": trt.__version__, + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(device), + "device": device, + "profile": 0, + } + + def __call__(self, feeds): + from .tensorrt_pair import buffers_for + + torch = self.torch + with torch.cuda.device(self.device), torch.cuda.stream(self.stream): + buffers, inputs = buffers_for(self.engine, self.context, feeds, torch, self.trt, self.device, False) + for name in inputs: + gpu, host = buffers[name] + gpu.copy_(host) + if not self.context.execute_async_v3(self.stream.cuda_stream): + raise RuntimeError("TensorRT dataset execution failed") + for name, (gpu, host) in buffers.items(): + if name not in inputs: + host.copy_(gpu) + self.stream.synchronize() + return {name: host.numpy() for name, (_, host) in buffers.items() if name not in inputs} + + +def pair_bundle(baseline, candidate): + old = json.loads(Path(baseline).read_text()) + for key in ("schema_version", "split", "levels", "samples"): + if old[key] != candidate[key]: + raise ValueError(f"Baseline bundle differs in held-out {key}") + return { + **{key: candidate[key] for key in ("schema_version", "split", "levels", "samples", "provenance")}, + "baseline": old["artifact"], + "candidate": candidate["artifact"], + } + + +def collect( + model, + manifest, + output, + backend="onnx", + provider="CPUExecutionProvider", + provider_options=None, + threads=1, + optimization="all", + device=0, + save_scores=False, + baseline_bundle=None, +): + if backend not in ("onnx", "tensorrt") or threads < 1 or device < 0 or optimization not in ("all", "disable"): + raise ValueError("Invalid backend, thread count, device or graph optimization") + metadata, digest = inference_manifest(manifest) + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "environment": {"platform": platform.platform(), "python": platform.python_version(), "numpy": np.__version__}, + "manifest": {"path": str(manifest), "sha256": digest, "contents": metadata}, + "settings": { + "backend": backend, + "provider": provider if backend == "onnx" else None, + "threads": threads if backend == "onnx" else None, + "optimization": optimization if backend == "onnx" else None, + "save_scores": save_scores, + }, + "batches": [], + "scope": ( + "Held-out prediction collection from explicit preprocessed inputs; " + "not speed, integer placement, score parity or deployment acceptance." + ), + } + try: + predictor = ( + OnnxPredictor(model, report, provider, provider_options or {}, threads, optimization) + if backend == "onnx" + else TensorRTPredictor(model, report, device) + ) + samples = {str(s["instance_id"]): s for s in metadata["samples"]} + path = output / "predictions.csv" + with path.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=COLUMNS) + writer.writeheader() + for index, batch in enumerate(metadata["batches"]): + feeds, record = load_batch(manifest, metadata, batch) + arrays = predictor(feeds) + rows = [] + for level, spec in enumerate(metadata["levels"]): + if spec["output"] not in arrays: + raise ValueError(f"Missing mapped output: {spec['output']}") + predicted, confidence = predictions(arrays[spec["output"]], spec, len(batch["sample_ids"])) + for identifier, pred, conf in zip(batch["sample_ids"], predicted, confidence, strict=True): + sample = samples[identifier] + rows.append( + dict( + instance_id=sample["instance_id"], + filename=sample["filename"], + level=level, + label=sample["labels"][level], + prediction=spec["classes"][int(pred)], + confidence=float(conf), + threshold=0, + ) + ) + if save_scores: + score_path = output / f"scores-{index:05d}.npz" + np.savez(score_path, **arrays) + record["scores"] = {"path": score_path.name, "sha256": file_hash(score_path)} + writer.writerows(rows) + report["batches"].append(record) + del feeds, arrays + artifact = { + "path": str(path.resolve()), + "sha256": file_hash(path), + "classes": [level["classes"] for level in metadata["levels"]], + "provenance": { + "model_files": report["model_files"], + "inference_manifest_sha256": digest, + "runner_sha256": report["runner_sha256"], + "runtime": report["runtime"], + "report": str((output / "report.json").resolve()), + }, + } + bundle = { + **{key: metadata[key] for key in ("schema_version", "split", "samples", "provenance")}, + "levels": [{key: spec[key] for key in ("name", "classes")} for spec in metadata["levels"]], + "artifact": artifact, + } + (output / "evaluation.json").write_text(json.dumps(bundle, indent=2) + "\n") + if baseline_bundle is not None: + (output / "comparison.json").write_text(json.dumps(pair_bundle(baseline_bundle, bundle), indent=2) + "\n") + report.update(status="inferred", predictions=artifact) + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--backend", choices=["onnx", "tensorrt"], default="onnx") + parser.add_argument("--provider", default="CPUExecutionProvider") + parser.add_argument("--provider-options", type=json.loads) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--optimization", choices=["all", "disable"], default="all") + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--save-scores", action="store_true") + parser.add_argument("--baseline-bundle", type=Path, help="Create comparison.json against a previous inference evaluation.json") + collect(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/inference_pair.py b/dev/benchmarks/inference/inference_pair.py new file mode 100644 index 00000000..42dd8e8b --- /dev/null +++ b/dev/benchmarks/inference/inference_pair.py @@ -0,0 +1,154 @@ +"""Run paired held-out inference and mini_metrics evaluation in fresh processes.""" + +import json +import os +import subprocess +import sys +from argparse import ArgumentParser +from pathlib import Path + +from .onnx_inference import file_hash + + +def run_pair(manifest, baseline, candidate, output, baseline_runtime=None, candidate_runtime=None, metrics_python=sys.executable): + """Compose maintained collectors without loading either runtime in this process. + + Runtime dictionaries accept a Python executable and collector options. Paths + are resolved from the caller's directory; child commands run from the checkout. + No dependencies are installed and existing output directories are never reused. + """ + manifest = Path(manifest).resolve(strict=True) + models = [Path(path).resolve(strict=True) for path in (baseline, candidate)] + commands = [] + output = Path(output).resolve() + for name, model, runtime in zip(("baseline", "candidate"), models, (baseline_runtime, candidate_runtime), strict=True): + options = dict(runtime or {}) + python = str(Path(options.pop("python", sys.executable)).absolute()) + allowed = {"backend", "provider", "provider_options", "threads", "optimization", "device", "save_scores"} + if options.keys() - allowed: + raise ValueError(f"Unknown {name} runtime options: {sorted(options.keys() - allowed)}") + command = [ + python, + "-m", + "dev.benchmarks.inference.dataset_inference", + "--model", + str(model), + "--manifest", + str(manifest), + "--output", + str(output / name), + ] + for key, value in options.items(): + if key == "save_scores": + if not isinstance(value, bool): + raise ValueError("save_scores must be a boolean") + if value: + command.append("--save-scores") + else: + command.extend(["--" + key.replace("_", "-"), json.dumps(value) if key == "provider_options" else str(value)]) + if name == "candidate": + command.extend(["--baseline-bundle", str(output / "baseline/evaluation.json")]) + commands.append((name, command, "inferred")) + commands.append( + ( + "quality", + [ + str(Path(metrics_python).absolute()), + "-m", + "dev.benchmarks.inference.quality_compare", + "--manifest", + str(output / "candidate/comparison.json"), + "--output", + str(output / "quality"), + ], + "evaluated", + ) + ) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "manifest": {"path": str(manifest), "sha256": file_hash(manifest)}, + "python_hash_seed": "0", + "stages": [], + "scope": "Paired held-out quality only; not timing, integer placement or production acceptance.", + } + report_path = output / "report.json" + + def save(): + report_path.write_text(json.dumps(report, indent=2) + "\n") + + try: + for name, command, expected in commands: + stage = {"name": name, "command": command, "status": "running", "log": f"{name}.log"} + report["stages"].append(stage) + save() + with (output / stage["log"]).open("w") as log: + result = subprocess.run( + command, + cwd=Path(__file__).resolve().parents[3], + env={**os.environ, "PYTHONHASHSEED": "0"}, + stdout=log, + stderr=subprocess.STDOUT, + check=False, + ) + stage["returncode"] = result.returncode + if result.returncode: + stage["status"] = "failed" + raise RuntimeError(f"{name} failed with exit code {result.returncode}; see {output / stage['log']}") + child_path = output / name / "report.json" + child = json.loads(child_path.read_text()) + if child["status"] != expected: + raise RuntimeError(f"Unexpected {name} report status: {child['status']}") + stage.update(status=expected, report_sha256=file_hash(child_path)) + save() + report.update(status="evaluated", levels=child["levels"], models=child["models"], undefined_metrics=child["undefined_metrics"]) + lines = [ + "# Paired inference quality", + "", + report["scope"], + "", + "| Level | Samples | Changed predictions | F1 delta | Recall delta | Precision delta | Coverage delta | Theil U delta |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for level in child["levels"]: + values = [level["candidate_minus_baseline"][metric] for metric in ("f1", "recall", "precision", "coverage", "theilU")] + label = str(level["name"]).replace("|", "\\|").replace("\n", " ").replace("\r", " ") + lines.append( + "| " + + " | ".join( + [ + label, + str(level["samples"]), + str(level["prediction_changes"]), + *["undefined" if v is None else f"{v:+.6f}" for v in values], + ] + ) + + " |" + ) + lines.extend(["", "Deltas are candidate minus baseline in metric units. Undefined values are not passes.", ""]) + (output / "summary.md").write_text("\n".join(lines)) + except BaseException as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + for stage in report["stages"]: + if stage["status"] == "running": + stage["status"] = "failed" + raise + finally: + save() + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + for name in ("manifest", "baseline", "candidate", "output"): + parser.add_argument("--" + name, type=Path, required=True) + for name in ("baseline", "candidate"): + parser.add_argument(f"--{name}-runtime", type=json.loads, help="JSON collector options and optional Python executable path") + parser.add_argument("--metrics-python", default=sys.executable, help="Explicit evaluation environment Python; installs nothing") + run_pair(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/onnx_calibration.py b/dev/benchmarks/inference/onnx_calibration.py new file mode 100644 index 00000000..0c2fe986 --- /dev/null +++ b/dev/benchmarks/inference/onnx_calibration.py @@ -0,0 +1,259 @@ +"""Calibrate a fresh ONNX QDQ artifact from explicit, ordered input batches.""" + +import hashlib +import inspect +import io +import json +import math +import platform +from argparse import ArgumentParser +from collections import Counter +from pathlib import Path + +import numpy as np + +from .onnx_inference import file_hash, model_files + + +def calibration_manifest(path): + manifest = json.loads(Path(path).read_text()) + if manifest.get("schema_version") != 1 or manifest.get("split") not in ("train", "calibration"): + raise ValueError("Require schema_version=1 and a declared train/calibration split") + if not isinstance(manifest.get("batch_input"), str) or not manifest["batch_input"]: + raise ValueError("Declare the input whose leading dimension identifies samples: batch_input") + if not isinstance(manifest.get("provenance"), dict) or not manifest["provenance"]: + raise ValueError("Record dataset/preprocessing provenance") + if not isinstance(manifest.get("batches"), list) or not manifest["batches"]: + raise ValueError("Supply an ordered nonempty batches list") + seen = set() + for batch in manifest["batches"]: + if not isinstance(batch.get("path"), str) or not batch["path"]: + raise ValueError("Each batch needs an NPZ path") + ids = batch.get("sample_ids") + if not isinstance(ids, list) or not ids or any(not isinstance(x, str) or not x for x in ids): + raise ValueError("Each batch needs nonempty string sample_ids") + if len(set(ids)) != len(ids) or seen.intersection(ids): + raise ValueError("Calibration sample IDs must be unique") + seen.update(ids) + return manifest + + +def load_batch(manifest_path, manifest, batch): + path = (Path(manifest_path).parent / batch["path"]).resolve() + payload = path.read_bytes() + digest = hashlib.sha256(payload).hexdigest() + if batch.get("sha256") is not None and batch["sha256"] != digest: + raise ValueError(f"Calibration input hash mismatch: {path}") + with np.load(io.BytesIO(payload), allow_pickle=False) as archive: + feeds = {name: archive[name].copy(order="C") for name in archive.files} + if not feeds or any(not np.isfinite(a).all() for a in feeds.values()): + raise ValueError(f"Calibration arrays must be finite: {path}") + name = manifest["batch_input"] + if name not in feeds or feeds[name].ndim == 0 or feeds[name].shape[0] != len(batch["sample_ids"]): + raise ValueError(f"sample_ids do not match the batch_input leading dimension: {path}") + record = { + "path": str(path), + "sha256": digest, + "sample_ids": batch["sample_ids"], + "arrays": {name: {"shape": list(a.shape), "dtype": str(a.dtype)} for name, a in feeds.items()}, + } + return feeds, record + + +def calibrate( + model, + manifest, + output, + method="percentile", + percentile=99.9, + activation_type="uint8", + symmetric_activations=False, + symmetric_calibration=False, + float_bias=False, + per_channel=True, + op_types=("Conv", "Gemm", "MatMul"), + threads=1, +): + if method not in ("minmax", "percentile") or activation_type not in ("uint8", "int8"): + raise ValueError("Choose minmax/percentile calibration and uint8/int8 activations") + if not math.isfinite(percentile) or not 0 < percentile <= 100 or threads < 1 or not op_types: + raise ValueError("Require percentile in (0,100], positive threads and selected operator types") + metadata = calibration_manifest(manifest) + try: + import onnx + import onnxruntime as ort + from onnxruntime.quantization import CalibrationDataReader, CalibrationMethod, QuantFormat, QuantType, quantize_static + from onnxruntime.quantization.calibrate import MinMaxCalibrater, PercentileCalibrater, save_tensors_data + except ImportError as error: + raise ImportError("Use an explicitly prepared ONNX/ONNX Runtime quantization environment; this command installs nothing") from error + if "calibration_cache_path" not in inspect.signature(quantize_static).parameters: + raise RuntimeError("This command requires ONNX Runtime's calibration-cache API (tested with 1.29.0)") + + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "versions": {"onnx": onnx.__version__, "onnxruntime": ort.__version__, "numpy": np.__version__}, + "environment": {"platform": platform.platform(), "python": platform.python_version()}, + "calibration_manifest": {"path": str(manifest), "sha256": file_hash(manifest), "contents": metadata}, + "recipe": { + "format": "QDQ", + "method": method, + "percentile": percentile if method == "percentile" else None, + "histogram_symmetric": symmetric_calibration, + "activation_type": activation_type, + "activation_symmetric": symmetric_activations, + "weight_type": "int8", + "weight_symmetric": True, + "per_channel_weights": per_channel, + "quantize_bias": not float_bias, + "op_types": list(op_types), + }, + "execution": { + "provider": "CPUExecutionProvider", + "intra_op_threads": threads, + "inter_op_threads": 1, + "calibration_graph_optimization": "disabled", + "collection_batch_limit": 1, + }, + "batches": [], + "scope": ( + "Calibration from declared training/calibration inputs and one calibration-batch execution; " + "not held-out quality, integer-kernel placement or deployment acceptance." + ), + } + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + base = MinMaxCalibrater if method == "minmax" else PercentileCalibrater + + class Calibrator(base): + def create_inference_session(self): + self.infer_session = ort.InferenceSession(self.augmented_model_path, sess_options=options, providers=["CPUExecutionProvider"]) + self.infer_session.disable_fallback() + + class OneBatch(CalibrationDataReader): + def __init__(self, feeds): + self.feeds = feeds + + def get_next(self): + feeds, self.feeds = self.feeds, None + return feeds + + try: + report["source_files"] = model_files(Path(model), onnx) + source = onnx.load(model) + report["source_ops"] = dict(Counter(node.op_type for node in source.graph.node)) + if not set(op_types).intersection(report["source_ops"]): + raise ValueError("No selected operator types occur in the source graph") + # ORT writes inferred-model sidecars beside its input. Isolate those writes. + snapshot_dir = output / "source" + snapshot_dir.mkdir() + snapshot = snapshot_dir / "model.onnx" + onnx.save_model( + source, snapshot, save_as_external_data=True, all_tensors_to_one_file=True, location="weights.data", size_threshold=0 + ) + del source + report["snapshot_files"] = model_files(snapshot, onnx) + collector = Calibrator( + snapshot, + list(op_types), + augmented_model_path=str(output / "calibration.onnx"), + use_external_data_format=True, + symmetric=symmetric_calibration, + **({"percentile": percentile} if method == "percentile" else {}), + ) + collector.augment_graph() + collector.create_inference_session() + names = {node.name for node in collector.infer_session.get_inputs()} + for entry in metadata["batches"]: + feeds, record = load_batch(manifest, metadata, entry) + report["batches"].append(record) + if set(feeds) != names: + raise ValueError("Calibration input names do not match the source model") + collector.collect_data(OneBatch(feeds)) + del feeds + cache = output / "ranges.json" + ranges = collector.compute_data() + if not len(ranges.data): + raise ValueError("No tensor ranges were calibrated") + report["calibrated_tensors"] = len(ranges.data) + save_tensors_data(ranges, cache) + del collector, ranges + report["ranges_sha256"] = file_hash(cache) + quantized = output / "model.onnx" + quantize_static( + snapshot, + quantized, + None, + quant_format=QuantFormat.QDQ, + per_channel=per_channel, + activation_type=QuantType.QUInt8 if activation_type == "uint8" else QuantType.QInt8, + weight_type=QuantType.QInt8, + op_types_to_quantize=list(op_types), + calibration_cache_path=cache, + use_external_data_format=True, + calibrate_method=CalibrationMethod.MinMax if method == "minmax" else CalibrationMethod.Percentile, + extra_options={"ActivationSymmetric": symmetric_activations, "WeightSymmetric": True, "QuantizeBias": not float_bias}, + ) + onnx.checker.check_model(str(quantized)) + graph = onnx.load(quantized, load_external_data=False) + report["output_ops"] = dict(Counter(node.op_type for node in graph.graph.node)) + if not report["output_ops"].get("QuantizeLinear") or not report["output_ops"].get("DequantizeLinear"): + raise ValueError("The output graph contains no complete QDQ quantization") + del graph + report["output_files"] = model_files(quantized, onnx) + # A smoke execution of calibration data establishes loadability, not quality. + feeds, record = load_batch(manifest, metadata, metadata["batches"][0]) + if record["sha256"] != report["batches"][0]["sha256"]: + raise ValueError("First calibration batch changed before smoke execution") + smoke_options = ort.SessionOptions() + smoke_options.intra_op_num_threads = threads + smoke_options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(quantized), sess_options=smoke_options, providers=["CPUExecutionProvider"]) + session.disable_fallback() + arrays = session.run(None, feeds) + if any(not np.isfinite(a).all() for a in arrays): + raise ValueError("Quantized graph produced nonfinite calibration-smoke outputs") + output_names = [node.name for node in session.get_outputs()] + np.savez(output / "calibration-smoke.npz", **dict(zip(output_names, arrays, strict=True))) + report["smoke"] = { + "batch_index": 0, + "graph_optimization": "all", + "outputs": output_names, + "sha256": file_hash(output / "calibration-smoke.npz"), + } + report["status"] = "passed" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True, help="Ordered calibration NPZ batches, sample IDs and provenance") + parser.add_argument("--output", type=Path, required=True, help="New artifact directory") + parser.add_argument("--method", choices=["minmax", "percentile"], default="percentile") + parser.add_argument("--percentile", type=float, default=99.9) + parser.add_argument("--activation-type", choices=["uint8", "int8"], default="uint8") + parser.add_argument("--symmetric-activations", action="store_true") + parser.add_argument("--symmetric-calibration", action="store_true", help="Collect symmetric calibration ranges/histograms") + parser.add_argument("--float-bias", action="store_true", help="Keep biases floating instead of quantizing them to INT32") + parser.add_argument("--per-tensor-weights", dest="per_channel", action="store_false") + parser.add_argument("--op-type", dest="op_types", action="append", help="Operator type to quantize; repeatable") + parser.add_argument("--threads", type=int, default=1) + args = vars(parser.parse_args()) + if args["op_types"] is None: + args["op_types"] = ["Conv", "Gemm", "MatMul"] + calibrate(**args) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/onnx_cpu_memory.py b/dev/benchmarks/inference/onnx_cpu_memory.py new file mode 100644 index 00000000..1c5cc65c --- /dev/null +++ b/dev/benchmarks/inference/onnx_cpu_memory.py @@ -0,0 +1,147 @@ +"""Measure one ONNX CPU model per fresh Linux process, without profiling.""" + +import json +import os +import platform +import statistics +import time +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np + +from .onnx_inference import file_hash, model_files + + +def resident_memory(): + """Linux mapping snapshot and approximate post-exec high-water mark, in bytes.""" + if platform.system() != "Linux": + raise RuntimeError("This memory probe requires Linux /proc") + fields = {} + for path, names in (("status", ("VmRSS", "VmHWM")), ("smaps_rollup", ("Rss", "Pss", "Swap"))): + for line in Path(f"/proc/self/{path}").read_text().splitlines(): + name, _, value = line.partition(":") + if name in names: + amount, unit = value.split() + if unit != "kB": + raise RuntimeError(f"Unexpected /proc memory unit: {unit}") + fields[name] = int(amount) * 1024 + return { + "resident_bytes": fields["Rss"], + "proportional_resident_bytes": fields["Pss"], + "swap_bytes": fields["Swap"], + "approximate_status_resident_bytes": fields["VmRSS"], + "peak_resident_bytes": fields["VmHWM"], + } + + +def measure(model, inputs, output, threads=1, warmup=3, repeats=31, optimization="all"): + """Use the CLI in a fresh process: in-process calls inherit earlier RSS peaks.""" + if min(threads, warmup, repeats) < 1 or optimization not in ("all", "disable"): + raise ValueError("Positive threads/warmup/repeats and all/disable optimization are required") + model, inputs = Path(model).resolve(strict=True), Path(inputs).resolve(strict=True) + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "environment": {"platform": platform.platform(), "machine": platform.machine(), "python": platform.python_version()}, + "settings": {"threads": threads, "inter_op_threads": 1, "warmup": warmup, "repeats": repeats, "optimization": optimization}, + "memory": {}, + "memory_sources": { + "resident_bytes": "/proc/self/smaps_rollup Rss", + "proportional_resident_bytes": "/proc/self/smaps_rollup Pss", + "swap_bytes": "/proc/self/smaps_rollup Swap", + "peak_resident_bytes": "/proc/self/status VmHWM (approximate)", + }, + "scope": ( + "Single CPU session, preprocessed NumPy IO, no profiling. RSS includes interpreter, imports, inputs and outputs. " + "Peak is the approximate high-water mark since exec through each snapshot, not model-only memory. " + "Memory includes finite-input/output validation. Snapshot reads occur outside inference timings. " + "Run in a fresh process for each model/trial. Snapshots precede ONNX artifact inspection and output serialization. " + "Session load is not guaranteed disk-cold; inference excludes image decoding and preprocessing." + ), + } + try: + report["memory"]["before_runtime_import"] = resident_memory() + import onnxruntime as ort + + report["versions"] = {"onnxruntime": ort.__version__, "numpy": np.__version__} + report["runtime_build"] = ort.get_build_info() + report["environment"]["cpu_affinity"] = sorted(os.sched_getaffinity(0)) + report["memory"]["after_runtime_import"] = resident_memory() + with np.load(inputs, allow_pickle=False) as archive: + feeds = {name: np.ascontiguousarray(archive[name]) for name in archive.files} + if not feeds or any(not np.isfinite(value).all() for value in feeds.values()): + raise ValueError("Inputs must contain finite named arrays") + report["input_arrays"] = {name: {"shape": list(a.shape), "dtype": str(a.dtype)} for name, a in feeds.items()} + report["memory"]["before_session"] = resident_memory() + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + options.graph_optimization_level = getattr( + ort.GraphOptimizationLevel, "ORT_ENABLE_ALL" if optimization == "all" else "ORT_DISABLE_ALL" + ) + started = time.perf_counter() + session = ort.InferenceSession(str(model), sess_options=options, providers=["CPUExecutionProvider"]) + report["session_load_seconds"] = time.perf_counter() - started + session.disable_fallback() + report["session_providers"] = session.get_providers() + report["memory"]["after_session"] = resident_memory() + if report["session_providers"] != ["CPUExecutionProvider"]: + raise RuntimeError("Expected an exclusively CPU session") + if set(feeds) != {node.name for node in session.get_inputs()}: + raise ValueError("Input names do not match model") + started = time.perf_counter() + predictions = session.run(None, feeds) + report["first_run_seconds"] = time.perf_counter() - started + report["memory"]["after_first_run"] = resident_memory() + del predictions + for _ in range(warmup): + session.run(None, feeds) + report["memory"]["after_warmup"] = resident_memory() + report["seconds"] = [] + for _ in range(repeats): + started = time.perf_counter() + predictions = session.run(None, feeds) + report["seconds"].append(time.perf_counter() - started) + # Do not retain the previous outputs while allocating the next batch. + if any(not np.isfinite(value).all() for value in predictions): + raise ValueError("Nonfinite model outputs") + del predictions + report["memory"]["after_measurement"] = resident_memory() + report["median_seconds"] = statistics.median(report["seconds"]) + # Provenance inspection can deserialize large inline tensors. Do it only + # after capturing the measurement peak, so it cannot inflate that result. + import onnx + + report["versions"]["onnx"] = onnx.__version__ + report["model_files"] = model_files(model, onnx) + report["inputs"] = {"path": str(inputs), "sha256": file_hash(inputs)} + predictions = session.run(None, feeds) + names = [node.name for node in session.get_outputs()] + np.savez(output / "outputs.npz", **dict(zip(names, predictions, strict=True))) + report["outputs"] = {"names": names, "sha256": file_hash(output / "outputs.npz")} + report["status"] = "measured" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + for name in ("model", "inputs", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--repeats", type=int, default=31) + parser.add_argument("--optimization", choices=["all", "disable"], default="all") + measure(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/onnx_inference.py b/dev/benchmarks/inference/onnx_inference.py new file mode 100644 index 00000000..ef8331b9 --- /dev/null +++ b/dev/benchmarks/inference/onnx_inference.py @@ -0,0 +1,179 @@ +"""Measure ONNX Session.run with explicit providers and portable preprocessed inputs.""" + +import hashlib +import json +import platform +import statistics +import time +from argparse import ArgumentParser +from collections import Counter +from pathlib import Path + +import numpy as np + + +def file_hash(path): + with Path(path).open("rb") as handle: + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def model_files(path, onnx): + """Include external tensor storage, also inside graph attributes/subgraphs.""" + paths = {path.resolve()} + + def visit(message): + if isinstance(message, onnx.TensorProto): + for entry in message.external_data: + if entry.key == "location": + paths.add((path.parent / entry.value).resolve()) + for field, value in message.ListFields(): + if field.message_type is not None: + for child in value if field.is_repeated else [value]: + visit(child) + + visit(onnx.load(path, load_external_data=False)) + return [{"path": str(p), "sha256": file_hash(p), "bytes": p.stat().st_size} for p in sorted(paths)] + + +def require_operations(counts, provider, operations): + """Require observed runtime operations to execute entirely on the chosen EP.""" + for operation in operations: + placement = {ep: count for (op, ep), count in counts.items() if op == operation and count} + if set(placement) != {provider}: + raise RuntimeError(f"Required operation {operation} must execute exclusively on {provider}; observed: {placement}") + + +def run(models, inputs, output, provider, threads=1, warmup=3, repeats=11, provider_options=None, required_ops=(), optimization="all"): + import onnx + import onnxruntime as ort + + levels = {"disable": "ORT_DISABLE_ALL", "basic": "ORT_ENABLE_BASIC", "extended": "ORT_ENABLE_EXTENDED", "all": "ORT_ENABLE_ALL"} + if optimization not in levels: + raise ValueError(f"Unknown graph optimization level: {optimization}") + if min(threads, warmup, repeats) < 1: + raise ValueError("threads, warmup and repeats must be positive") + if not models or len(set(models)) != len(models): + raise ValueError("Supply distinct model paths") + if provider not in ort.get_available_providers(): + raise ValueError(f"Requested provider {provider} unavailable; available: {ort.get_available_providers()}") + with np.load(inputs, allow_pickle=False) as archive: + feeds = {name: np.ascontiguousarray(archive[name]) for name in archive.files} + if not feeds or any(not np.isfinite(value).all() for value in feeds.values()): + raise ValueError("Inputs must contain finite named arrays") + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "environment": {"platform": platform.platform(), "machine": platform.machine(), "python": platform.python_version()}, + "versions": {"onnx": onnx.__version__, "onnxruntime": ort.__version__, "numpy": np.__version__}, + "runtime_build": ort.get_build_info(), + "runner_sha256": file_hash(__file__), + "provider": provider, + "provider_options": provider_options or {}, + "required_ops": list(required_ops), + "optimization": optimization, + "available_providers": ort.get_available_providers(), + "threads": threads, + "warmup": warmup, + "repeats": repeats, + "inputs": {"path": str(inputs), "sha256": file_hash(inputs)}, + "input_arrays": {name: {"shape": list(a.shape), "dtype": str(a.dtype)} for name, a in feeds.items()}, + "scope": ( + "Warm Session.run with CPU NumPy inputs and outputs; includes device transfers, " + "excludes image IO, preprocessing and session construction." + ), + "order": "alternating forward/reverse model order per repetition", + "models": [], + } + sessions = [] + providers = [(provider, provider_options or {})] + if provider != "CPUExecutionProvider": + providers.append("CPUExecutionProvider") + + def options(): + opts = ort.SessionOptions() + opts.intra_op_num_threads = threads + opts.inter_op_num_threads = 1 + opts.graph_optimization_level = getattr(ort.GraphOptimizationLevel, levels[optimization]) + return opts + + try: + for index, path in enumerate(map(Path, models)): + record = {"path": str(path), "files": model_files(path, onnx), "seconds": []} + report["models"].append(record) + opts = options() + opts.enable_profiling = True + opts.profile_file_prefix = str(output / f"model-{index}-profile") + session = ort.InferenceSession(str(path), sess_options=opts, providers=providers) + session.disable_fallback() + try: + if set(feeds) != {node.name for node in session.get_inputs()}: + raise ValueError(f"Input names do not match {path}") + predictions = session.run(None, feeds) + finally: + profile = session.end_profiling() + counts = Counter( + (event["args"].get("op_name"), event["args"].get("provider")) + for event in json.loads(Path(profile).read_text()) + if event.get("cat") == "Node" and event.get("args", {}).get("provider") + ) + record["execution"] = [{"op": op, "provider": ep, "count": n} for (op, ep), n in sorted(counts.items())] + record["profile"] = str(profile) + if not any(ep == provider for _, ep in counts): + raise RuntimeError(f"No profiled operation executed on requested provider {provider}") + require_operations(counts, provider, required_ops) + if any(not np.isfinite(array).all() for array in predictions): + raise ValueError(f"Nonfinite predictions from {path}") + record["outputs"] = [node.name for node in session.get_outputs()] + np.savez(output / f"model-{index}-outputs.npz", **dict(zip(record["outputs"], predictions, strict=True))) + del session + session = ort.InferenceSession(str(path), sess_options=options(), providers=providers) + session.disable_fallback() + record["session_providers"] = session.get_providers() + record["session_provider_options"] = session.get_provider_options() + sessions.append(session) + for trial in range(warmup + repeats): + order = range(len(sessions)) if trial % 2 else reversed(range(len(sessions))) + for index in order: + started = time.perf_counter() + sessions[index].run(None, feeds) + elapsed = time.perf_counter() - started + if trial >= warmup: + report["models"][index]["seconds"].append(elapsed) + for record in report["models"]: + record["median_seconds"] = statistics.median(record["seconds"]) + report["status"] = "passed" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, action="append", required=True) + parser.add_argument("--inputs", type=Path, required=True, help="NPZ with exact ONNX input names and preprocessed arrays") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--provider", required=True) + parser.add_argument("--provider-options", type=json.loads, default={}) + parser.add_argument("--optimization", choices=["disable", "basic", "extended", "all"], default="all") + parser.add_argument( + "--require-provider-op", + dest="required_ops", + action="append", + default=[], + help="Require this profiled operation type to execute entirely on the requested provider (repeatable)", + ) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--repeats", type=int, default=11) + args = vars(parser.parse_args()) + args["models"] = args.pop("model") + run(**args) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/prepare_inputs.py b/dev/benchmarks/inference/prepare_inputs.py new file mode 100644 index 00000000..e2548d40 --- /dev/null +++ b/dev/benchmarks/inference/prepare_inputs.py @@ -0,0 +1,301 @@ +"""Prepare calibration or held-out image batches from explicit export/dataset contracts.""" + +import hashlib +import importlib +import inspect +import json +import random +from argparse import ArgumentParser +from contextlib import contextmanager +from pathlib import Path + +import numpy as np + +from .dataset_inference import inference_manifest +from .onnx_calibration import calibration_manifest +from .onnx_inference import file_hash + + +@contextmanager +def seeded(seed): + """Restore caller RNGs; factories and transforms get explicit independent seeds.""" + import torch + + python_state, numpy_state = random.getstate(), np.random.get_state() + try: + with torch.random.fork_rng(devices=[]): + random.seed(seed) + np.random.seed(seed % 2**32) + torch.manual_seed(seed) + yield + finally: + random.setstate(python_state) + np.random.set_state(numpy_state) + + +def ordered_classes(mapping): + if not isinstance(mapping, dict) or not mapping or any(not isinstance(k, str) or not k for k in mapping): + raise ValueError("Require nonempty class-name mappings") + if all(type(v) is int for v in mapping.values()): + if sorted(mapping.values()) != list(range(len(mapping))): + raise ValueError("Class indices must be unique and contiguous from zero") + return [sorted(mapping, key=mapping.get)] + if set(mapping) != {str(i) for i in range(len(mapping))}: + raise ValueError("Hierarchical level indices must be contiguous from zero") + result = [] + for i in range(len(mapping)): + level = ordered_classes(mapping[str(i)]) + if len(level) != 1: + raise ValueError("Expected one class mapping per level") + result.extend(level) + return result + + +def select_records(dataset, split, count, seed): + if dataset.get("schema_version") != 1 or split not in ("train", "val", "test"): + raise ValueError("Require a version-1 dataset and train/val/test split") + seen, hashes, selected = set(), {}, [] + for record in dataset["records"]: + path, digest, source_split = record["path"], record["sha256"], record["split"] + if not isinstance(path, str) or not path or path in seen: + raise ValueError("Dataset paths must be nonempty and unique") + seen.add(path) + if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError("Require lowercase SHA256 source image hashes") + if source_split not in ("train", "val", "test"): + raise ValueError("Unknown dataset split") + if digest in hashes and hashes[digest] != source_split: + raise ValueError("Dataset declares byte-identical images across splits") + hashes[digest] = source_split + if source_split == split: + selected.append(record) + if not selected: + raise ValueError("Selected split is empty") + if count is not None: + if not 0 < count <= len(selected): + raise ValueError("count must fit the selected split") + selected = random.Random(seed).sample(selected, count) + return selected + + +def repository_preprocess(metadata, model_args=None): + """Resolve preprocessing through existing loaders without constructing the trained head.""" + from mini_trainer.modeling.architectures.load import get_dynamic_model, get_model, resolve_backbone_getter + from mini_trainer.utils import string_to_dtype + + name = metadata["backbone_class"] + getter, _ = resolve_backbone_getter(name) + args = {} if getter is get_dynamic_model else {"pretrained": False, "local_files_only": True} + args.update(model_args or {}) + args["resize_size"] = metadata["resize_size"] + dtype = string_to_dtype(metadata.get("preprocess_dtype") or metadata.get("_dtype", "float32")) + _, _, preprocess, _, _ = get_model(name, model_args=args, preprocess_dtype=dtype) + return preprocess + + +def prepare( + dataset_manifest, + data_root, + export_manifest, + output, + split, + count=None, + seed=42, + batch_size=8, + source_levels=None, + level_names=None, + classifier_module=None, + preprocess_factory="dev.benchmarks.inference.prepare_inputs:repository_preprocess", + factory_args=None, + threads=1, + score_semantics=None, +): + import torch + + from mini_trainer.data import get_inference_dataloader + + if batch_size < 1 or threads < 1: + raise ValueError("Require positive batch size and thread count") + dataset_bytes, export_bytes = Path(dataset_manifest).read_bytes(), Path(export_manifest).read_bytes() + dataset, exported = json.loads(dataset_bytes), json.loads(export_bytes) + records = select_records(dataset, split, count, seed) + heads = exported["classifiers"] + if classifier_module is not None: + heads = [head for head in heads if head["module"] == classifier_module] + if len(heads) != 1: + raise ValueError("Select one unambiguous classifier metadata entry with classifier_module") + metadata = heads[0]["metadata"] + classes = ordered_classes(metadata["cls2idx"]) + if score_semantics is None: + raise ValueError("Declare logits/probabilities score semantics explicitly") + semantics = [score_semantics] if isinstance(score_semantics, str) else list(score_semantics) + if len(semantics) == 1: + semantics *= len(classes) + if len(semantics) != len(classes) or any(s not in ("logits", "probabilities") for s in semantics): + raise ValueError("Supply score semantics for every exported level") + dataset_classes = ordered_classes(dataset["class_spec"]["cls2idx"]) + source_levels = list(range(len(classes))) if source_levels is None else source_levels + if ( + len(source_levels) != len(classes) + or len(set(source_levels)) != len(source_levels) + or any(type(i) is not int or not 0 <= i < len(dataset_classes) for i in source_levels) + ): + raise ValueError("Supply one distinct valid source level per exported classifier level") + if [dataset_classes[i] for i in source_levels] != classes: + raise ValueError("Export class ordering differs from the selected dataset levels") + names = [f"level_{i}" for i in range(len(classes))] if level_names is None else level_names + if len(names) != len(classes) or len(set(names)) != len(names) or any(not isinstance(n, str) or not n for n in names): + raise ValueError("Supply unique names for every exported level") + outputs = exported["outputs"] + if len(outputs) != len(classes): + raise ValueError("Export outputs do not match the selected classifier's levels") + if exported["preprocessing"]["in_graph"]: + raise ValueError("This preparation command expects preprocessing outside the graph") + input_spec = exported["input"] + if input_spec["dtype"] not in ("float16", "float32", "float64"): + raise ValueError("This image preparation path requires a NumPy-compatible floating ONNX input") + if "resize_size" not in metadata: + raise ValueError("Export metadata must declare the repository image-reader resize_size") + root = Path(data_root).resolve() + paths = [(root / record["path"]).resolve() for record in records] + if any(not path.is_relative_to(root) for path in paths): + raise ValueError("Source images must remain under data_root") + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "dataset_manifest_sha256": hashlib.sha256(dataset_bytes).hexdigest(), + "export_manifest_sha256": hashlib.sha256(export_bytes).hexdigest(), + "export_source": exported.get("source"), + "classifier_metadata": metadata, + "selection": { + "split": split, + "count": count, + "seed": seed, + "method": "manifest order" if count is None else "random.Random(seed).sample", + }, + "settings": {"batch_size": batch_size, "threads": threads, "workers": 0, "source_levels": source_levels}, + "source_records": records, + "batches": [], + "scope": "Explicit image preprocessing and input artifacts; not calibration, model inference or target-hardware acceptance.", + } + previous_threads = torch.get_num_threads() + try: + torch.set_num_threads(threads) + for path, record in zip(paths, records, strict=True): + if file_hash(path) != record["sha256"]: + raise ValueError(f"Source image hash mismatch: {path}") + factory = preprocess_factory + if isinstance(factory, str): + module, name = factory.split(":", 1) + factory = getattr(importlib.import_module(module), name) + factory_source = inspect.getsourcefile(factory) + report["preprocessing"] = { + "factory": f"{factory.__module__}:{factory.__qualname__}", + "factory_args": factory_args or {}, + "factory_source_sha256": file_hash(factory_source) if factory_source else None, + "torch": torch.__version__, + "numpy": np.__version__, + } + with seeded(seed): + preprocess = factory(metadata, **(factory_args or {})) + report["preprocessing"]["description"] = repr(preprocess) + _, loader = get_inference_dataloader( + images=[str(path) for path in paths], + resize_size=metadata["resize_size"], + batch_size=batch_size, + num_workers=0, + device="cpu", + dtype=torch.float32, + ) + samples = [] + for index, record in enumerate(records): + targets = record["targets"] + labels = [] + for classes_at_level, source_level in zip(classes, source_levels, strict=True): + target = targets[source_level] + if type(target) is not int or not 0 <= target < len(classes_at_level): + raise ValueError("Dataset target index is outside the declared class mapping") + labels.append(classes_at_level[target]) + samples.append({"instance_id": index, "filename": record["path"], "labels": labels}) + offset = 0 + with torch.inference_mode(), seeded(seed): + for index, images in enumerate(loader): + transformed = preprocess(images) + if not isinstance(transformed, torch.Tensor): + raise ValueError("Image preprocessing must return one tensor for the exported image input") + array = transformed.detach().cpu().to(getattr(torch, input_spec["dtype"])).numpy() + shape = input_spec["shape"] + if array.ndim != len(shape) or any(type(n) is int and n != actual for n, actual in zip(shape, array.shape, strict=True)): + raise ValueError("Prepared tensor shape does not match the exported input") + if len(array) != len(images) or not np.isfinite(array).all(): + raise ValueError("Preprocessing must preserve batch identity and produce finite values") + path = output / f"batch-{index:05d}.npz" + np.savez(path, **{input_spec["name"]: array}) + ids = [str(i) for i in range(offset, offset + len(array))] + report["batches"].append({"path": path.name, "sha256": file_hash(path), "sample_ids": ids}) + offset += len(array) + if offset != len(records): + raise ValueError("Preparation did not cover all selected records") + for path, record in zip(paths, records, strict=True): + if file_hash(path) != record["sha256"]: + raise ValueError(f"Source image changed during preparation: {path}") + manifest = { + "schema_version": 1, + "split": split, + "provenance": { + key: report[key] + for key in ( + "dataset_manifest_sha256", + "export_manifest_sha256", + "export_source", + "selection", + "preprocessing", + "runner_sha256", + ) + }, + "batch_input": input_spec["name"], + "batches": report["batches"], + "levels": [ + {"name": name, "classes": cls, "output": binding["name"], "score_semantics": semantics[i]} + for i, (name, cls, binding) in enumerate(zip(names, classes, outputs, strict=True)) + ], + "samples": samples, + } + manifest_path = output / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + (calibration_manifest if split == "train" else inference_manifest)(manifest_path) + report.update(status="prepared", manifest_sha256=file_hash(manifest_path)) + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + torch.set_num_threads(previous_threads) + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--dataset-manifest", type=Path, required=True) + parser.add_argument("--data-root", type=Path, required=True) + parser.add_argument("--export-manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--split", choices=["train", "val", "test"], required=True) + parser.add_argument("--count", type=int) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--source-levels", type=int, nargs="+") + parser.add_argument("--level-names", nargs="+") + parser.add_argument("--classifier-module") + parser.add_argument("--preprocess-factory", default="dev.benchmarks.inference.prepare_inputs:repository_preprocess") + parser.add_argument("--factory-args", type=json.loads) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--score-semantics", choices=["logits", "probabilities"], nargs="+", required=True) + prepare(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/quality_compare.py b/dev/benchmarks/inference/quality_compare.py new file mode 100644 index 00000000..d5d8618c --- /dev/null +++ b/dev/benchmarks/inference/quality_compare.py @@ -0,0 +1,226 @@ +"""Compare paired prediction tables with explicit held-out identity and mini_metrics.""" + +import csv +import hashlib +import importlib.metadata +import io +import json +import math +import os +import platform +from argparse import ArgumentParser +from pathlib import Path + +from .onnx_inference import file_hash + +METRICS = ("f1", "recall", "precision", "coverage", "theilU") +COLUMNS = ("instance_id", "filename", "level", "label", "prediction", "confidence", "threshold") + + +def read_manifest(path): + payload = Path(path).read_bytes() + manifest = json.loads(payload) + validate_dataset(manifest) + for mode in ("baseline", "candidate"): + artifact = manifest.get(mode, {}) + if not isinstance(artifact.get("path"), str) or not artifact["path"] or not artifact.get("provenance"): + raise ValueError(f"Declare prediction path and model/preprocessing provenance for {mode}") + if artifact.get("classes") != [level["classes"] for level in manifest["levels"]]: + raise ValueError(f"{mode} class mappings must match the ordered level mappings") + return manifest, hashlib.sha256(payload).hexdigest() + + +def validate_dataset(manifest): + """Validate the shared held-out identity contract before inference or evaluation.""" + if manifest.get("schema_version") != 1 or manifest.get("split") not in ("val", "test"): + raise ValueError("Require schema_version=1 and a declared val/test split") + if not isinstance(manifest.get("provenance"), dict) or not manifest["provenance"]: + raise ValueError("Record dataset and preprocessing provenance") + levels = manifest.get("levels") + if not isinstance(levels, list) or not levels: + raise ValueError("Supply ordered level names and class mappings") + names = set() + for level in levels: + name, classes = level.get("name"), level.get("classes") + if not isinstance(name, str) or not name or name in names: + raise ValueError("Level names must be nonempty and unique") + names.add(name) + if ( + not isinstance(classes, list) + or not classes + or any(not isinstance(c, str) or not c for c in classes) + or len(set(classes)) != len(classes) + ): + raise ValueError("Each level requires unique nonempty class names in score-column order") + samples = manifest.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("Declare the complete held-out sample set") + ids = set() + class_sets = [set(level["classes"]) for level in levels] + for sample in samples: + identifier = sample.get("instance_id") + if type(identifier) is not int or not 0 <= identifier < 2**63 or identifier in ids: + raise ValueError("Sample IDs must be unique nonnegative INT64 integers") + ids.add(identifier) + if not isinstance(sample.get("filename"), str) or not sample["filename"]: + raise ValueError("Each sample needs its filename/identity") + labels = sample.get("labels") + if not isinstance(labels, list) or len(labels) != len(levels): + raise ValueError("Each sample needs one label per level") + if any(label not in classes for label, classes in zip(labels, class_sets, strict=True)): + raise ValueError("Sample labels must belong to the declared classes") + + +def read_predictions(path, artifact, manifest): + payload = Path(path).read_bytes() + digest = hashlib.sha256(payload).hexdigest() + if artifact.get("sha256") is not None and artifact["sha256"] != digest: + raise ValueError(f"Prediction hash mismatch: {path}") + reader = csv.DictReader(io.StringIO(payload.decode("utf-8-sig"))) + if reader.fieldnames is None or set(reader.fieldnames) != set(COLUMNS) or len(reader.fieldnames) != len(COLUMNS): + raise ValueError("Prediction CSV must contain exactly the documented seven columns") + samples = {sample["instance_id"]: sample for sample in manifest["samples"]} + classes = [set(level["classes"]) for level in manifest["levels"]] + rows = {} + for row in reader: + if None in row or any(value is None for value in row.values()): + raise ValueError("Malformed prediction CSV row") + identifier, level = int(row["instance_id"]), int(row["level"]) + key = identifier, level + if identifier not in samples or not 0 <= level < len(classes) or key in rows: + raise ValueError("Unknown or duplicate sample/level in predictions") + sample = samples[identifier] + if row["filename"] != sample["filename"] or row["label"] != sample["labels"][level]: + raise ValueError("Prediction filename/label does not match held-out manifest") + if row["prediction"] not in classes[level]: + raise ValueError("Prediction is outside declared classes") + confidence, threshold = float(row["confidence"]), float(row["threshold"]) + if not math.isfinite(confidence) or not 0 <= confidence <= 1 or threshold != 0: + raise ValueError("Require confidence in [0,1] and threshold zero; this comparison does not tune or abstain") + rows[key] = {**row, "instance_id": identifier, "level": level, "confidence": confidence, "threshold": 0.0} + if len(rows) != len(samples) * len(classes): + raise ValueError("Predictions do not cover every held-out sample at every level") + # Canonicalize rows so CSV order cannot silently change the comparison. + ordered = [rows[(sample["instance_id"], level)] for level in range(len(classes)) for sample in manifest["samples"]] + return {column: [row[column] for row in ordered] for column in COLUMNS}, digest + + +def compare(manifest, output): + metadata, digest = read_manifest(manifest) + try: + import mini_metrics + from mini_metrics.data import MetricDF + from mini_metrics.metrics import MacroF1, evaluate_file + except ImportError as error: + raise ImportError("Prepare mini_metrics explicitly in the evaluation environment; this command installs nothing") from error + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "environment": { + "python": platform.python_version(), + "python_hash_seed": os.environ.get("PYTHONHASHSEED"), + }, + "manifest": {"path": str(manifest), "sha256": digest, "contents": metadata}, + "policy": { + "threshold": 0, + "optimal": False, + "known_only": False, + "per_class": False, + "criterion": "MacroF1", + "hierarchical_metrics": False, + }, + "scope": ( + "Fixed predictions evaluated independently at each declared level; " + "not threshold tuning, score parity, inference performance or production acceptance." + ), + "models": {}, + "undefined_metrics": [], + } + try: + root = Path(mini_metrics.__file__).parent + try: + version = importlib.metadata.version("mini_metrics") + except importlib.metadata.PackageNotFoundError: + version = None + report["mini_metrics"] = { + "installed_distribution_version": version, + "imported_package": str(root), + "source_hashes": {str(path.relative_to(root)): file_hash(path) for path in sorted(root.rglob("*.py"))}, + } + tables = {} + for mode in ("baseline", "candidate"): + artifact = metadata[mode] + path = Path(manifest).parent / artifact["path"] + table, source_hash = read_predictions(path, artifact, metadata) + tables[mode] = table + info = {"source_sha256": source_hash, "source_path": str(path)} + report["models"][mode] = info + normalized = output / f"{mode}.csv" + with normalized.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=COLUMNS) + writer.writeheader() + writer.writerows(dict(zip(COLUMNS, row, strict=True)) for row in zip(*(table[c] for c in COLUMNS), strict=True)) + info["normalized_csv_sha256"] = file_hash(normalized) + # Construct directly to preserve literal string labels such as "001". + result = evaluate_file( + MetricDF(table), + optimal=False, + threshold=0, + known_only=False, + per_class=False, + simple=True, + hierarchical=False, + pattern=r"^(f1|recall|precision|coverage|theilU)$", + opt_crit=MacroF1, + verbose=0, + ) + if set(result) != set(METRICS): + raise ValueError("mini_metrics did not return exactly the requested five metrics") + info["metrics"] = {} + for metric in METRICS: + values = {int(level): float(value) for level, value in result[metric].items()} + if set(values) != set(range(len(metadata["levels"]))): + raise ValueError("mini_metrics returned unexpected levels") + info["metrics"][metric] = {str(level): value if math.isfinite(value) else None for level, value in values.items()} + report["undefined_metrics"].extend( + {"model": mode, "metric": metric, "level": level} for level, value in values.items() if not math.isfinite(value) + ) + report["levels"] = [] + for level, spec in enumerate(metadata["levels"]): + delta = {} + for metric in METRICS: + a, b = [report["models"][mode]["metrics"][metric][str(level)] for mode in ("baseline", "candidate")] + delta[metric] = None if a is None or b is None else b - a + predictions = [ + [pred for pred, lvl in zip(tables[mode]["prediction"], tables[mode]["level"], strict=True) if lvl == level] + for mode in ("baseline", "candidate") + ] + report["levels"].append( + { + "name": spec["name"], + "samples": len(metadata["samples"]), + "candidate_minus_baseline": delta, + "prediction_changes": sum(a != b for a, b in zip(*predictions, strict=True)), + } + ) + report["status"] = "evaluated" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True, help="New directory for canonical CSVs and metric report") + compare(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/tensorrt_build.py b/dev/benchmarks/inference/tensorrt_build.py new file mode 100644 index 00000000..b0e8a5c4 --- /dev/null +++ b/dev/benchmarks/inference/tensorrt_build.py @@ -0,0 +1,239 @@ +"""Build, inspect and smoke-test an ONNX TensorRT engine in an explicit environment.""" + +import json +import math +import platform +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np + +from .onnx_inference import file_hash, model_files + + +def input_profiles(feeds, profiles=None): + """Validate explicit ranges; absent ranges use the supplied fixed input shapes.""" + if profiles is None: + profiles = {name: {key: list(value.shape) for key in ("min", "opt", "max")} for name, value in feeds.items()} + if set(profiles) != set(feeds): + raise ValueError("Profile names must match the named input arrays") + result = {} + for name, value in feeds.items(): + ranges = profiles[name] + if set(ranges) != {"min", "opt", "max"}: + raise ValueError(f"Profile {name} needs min, opt and max shapes") + for shape in ranges.values(): + if len(shape) != value.ndim or any(type(n) is not int or n < 1 for n in shape): + raise ValueError(f"Invalid dimensions in profile {name}") + for low, opt, high, sample in zip(ranges["min"], ranges["opt"], ranges["max"], value.shape, strict=True): + if not low <= opt <= high or not low <= sample <= high: + raise ValueError(f"Unordered ranges or sample outside profile {name}") + result[name] = {key: list(shape) for key, shape in ranges.items()} + return result + + +def compare_outputs(actual, expected, rtol, atol): + if set(actual) != set(expected): + raise ValueError("Reference output names do not match engine outputs") + result = {} + for name, value in actual.items(): + reference = expected[name] + if value.shape != reference.shape or value.dtype != reference.dtype: + raise ValueError(f"Reference shape/dtype does not match output {name}") + if not np.isfinite(reference).all(): + raise ValueError(f"Nonfinite reference output {name}") + close = np.isclose(value, reference, rtol=rtol, atol=atol) if value.dtype.kind == "f" else value == reference + result[name] = {"passed": bool(close.all()), "elements_outside_tolerance": int((~close).sum())} + return result + + +def build( + model, + inputs, + output, + profiles=None, + fp16=False, + tf32=False, + workspace_mib=1024, + optimization=3, + device=0, + reference=None, + rtol=1e-4, + atol=1e-5, +): + if workspace_mib < 1 or not 0 <= optimization <= 5 or device < 0: + raise ValueError("Require positive workspace, optimization 0–5 and nonnegative device index") + if any(not math.isfinite(x) or x < 0 for x in (rtol, atol)): + raise ValueError("Parity tolerances must be finite and nonnegative") + with np.load(inputs, allow_pickle=False) as data: + feeds = {name: data[name].copy(order="C") for name in data.files} + if not feeds or any(not np.isfinite(a).all() for a in feeds.values()): + raise ValueError("Supply finite named input arrays") + profiles = input_profiles(feeds, profiles) + try: + import onnx + import tensorrt as trt + import torch + except ImportError as error: + raise ImportError( + "Use an explicitly prepared compatible TensorRT, ONNX and CUDA PyTorch environment; no packages are installed by this command" + ) from error + + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "versions": {"tensorrt": trt.__version__, "torch": torch.__version__, "onnx": onnx.__version__, "numpy": np.__version__}, + "environment": {"platform": platform.platform(), "python": platform.python_version()}, + "settings": { + "profiles": profiles, + "fp16_allowed": fp16, + "tf32_allowed": tf32, + "workspace_bytes": workspace_mib * 1024**2, + "builder_optimization": optimization, + "device": device, + }, + "inputs": { + "path": str(inputs), + "sha256": file_hash(inputs), + "arrays": {name: {"shape": list(a.shape), "dtype": str(a.dtype)} for name, a in feeds.items()}, + }, + "reference": None, + "messages": [], + "scope": ( + "Engine build, detailed layer inspection and one input execution; not a quality or performance certification. " + "Workspace/context bytes are not total GPU memory." + ), + } + + class Logger(trt.ILogger): + def log(self, severity, message): + if severity <= trt.ILogger.WARNING: + report["messages"].append({"severity": str(severity), "message": message}) + + try: + report["model_files"] = model_files(Path(model), onnx) + with torch.cuda.device(device): + report["environment"]["gpu"] = torch.cuda.get_device_name(device) + report["environment"]["compute_capability"] = list(torch.cuda.get_device_capability(device)) + logger = Logger() + trt.init_libnvinfer_plugins(logger, "") + builder = trt.Builder(logger) + network = builder.create_network(0) + parser = trt.OnnxParser(network, logger) + if not parser.parse_from_file(str(model)): + report["parser_errors"] = [str(parser.get_error(i)) for i in range(parser.num_errors)] + raise RuntimeError("TensorRT could not parse the model; see parser_errors in report.json") + tensors = {network.get_input(i).name: network.get_input(i) for i in range(network.num_inputs)} + if set(tensors) != set(feeds): + raise ValueError("Input names do not match the parsed network") + profile = builder.create_optimization_profile() + for name, tensor in tensors.items(): + if tensor.is_shape_tensor: + raise ValueError(f"Shape-tensor input profiles are not supported by this probe: {name}") + if np.dtype(trt.nptype(tensor.dtype)) != feeds[name].dtype: + raise ValueError(f"Input dtype does not match network: {name}") + for shape in profiles[name].values(): + if len(shape) != len(tensor.shape) or any(n >= 0 and n != s for n, s in zip(tensor.shape, shape, strict=True)): + raise ValueError(f"Profile changes a fixed network dimension: {name}") + profile.set_shape(name, profiles[name]["min"], profiles[name]["opt"], profiles[name]["max"]) + config = builder.create_builder_config() + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace_mib * 1024**2) + config.builder_optimization_level = optimization + config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED + config.clear_flag(trt.BuilderFlag.TF32) + if tf32: + config.set_flag(trt.BuilderFlag.TF32) + if fp16: + config.set_flag(trt.BuilderFlag.FP16) + if config.add_optimization_profile(profile) < 0: + raise ValueError("TensorRT rejected the optimization profile") + serialized = builder.build_serialized_network(network, config) + if serialized is None: + raise RuntimeError("TensorRT engine construction failed; see messages in report.json") + engine_path = output / "model.engine" + engine_path.write_bytes(serialized) + runtime = trt.Runtime(logger) + engine = runtime.deserialize_cuda_engine(serialized) + if engine is None: + raise RuntimeError("TensorRT could not deserialize the constructed engine") + layers_path = output / "layers.json" + layers_path.write_text(engine.create_engine_inspector().get_engine_information(trt.LayerInformationFormat.JSON)) + report["engine"] = { + "sha256": file_hash(engine_path), + "bytes": engine_path.stat().st_size, + "context_memory_bytes": engine.device_memory_size_v2, + "layers_sha256": file_hash(layers_path), + "num_layers": engine.num_layers, + } + context = engine.create_execution_context() + stream = torch.cuda.Stream(device=device) + buffers, outputs = {}, {} + with torch.cuda.stream(stream): + for name, value in feeds.items(): + buffers[name] = torch.from_numpy(value).to(device=f"cuda:{device}") + if not context.set_input_shape(name, value.shape): + raise ValueError(f"TensorRT rejected sample shape for {name}") + for index in range(engine.num_io_tensors): + name = engine.get_tensor_name(index) + if ( + engine.get_tensor_format(name) != trt.TensorFormat.LINEAR + or engine.get_tensor_location(name) != trt.TensorLocation.DEVICE + ): + raise ValueError(f"Probe requires linear device IO tensors: {name}") + if engine.get_tensor_mode(name) == trt.TensorIOMode.OUTPUT: + shape = tuple(context.get_tensor_shape(name)) + if any(n <= 0 for n in shape): + raise ValueError(f"Unresolved or empty output shape for {name}: {shape}") + dtype = torch.from_numpy(np.empty((), dtype=trt.nptype(engine.get_tensor_dtype(name)))).dtype + buffers[name] = torch.empty(shape, device=f"cuda:{device}", dtype=dtype) + outputs[name] = buffers[name] + if not context.set_tensor_address(name, buffers[name].data_ptr()): + raise RuntimeError(f"TensorRT rejected IO address for {name}") + if not context.execute_async_v3(stream.cuda_stream): + raise RuntimeError("TensorRT execution failed") + stream.synchronize() + actual = {name: value.cpu().numpy() for name, value in outputs.items()} + if any(not np.isfinite(a).all() for a in actual.values()): + raise ValueError("Engine produced nonfinite outputs") + np.savez(output / "outputs.npz", **actual) + report["outputs"] = {name: {"shape": list(a.shape), "dtype": str(a.dtype)} for name, a in actual.items()} + if reference is not None: + report["reference"] = {"path": str(reference), "sha256": file_hash(reference), "rtol": rtol, "atol": atol} + with np.load(reference, allow_pickle=False) as expected: + report["reference"]["comparison"] = compare_outputs(actual, expected, rtol, atol) + if not all(r["passed"] for r in report["reference"]["comparison"].values()): + raise ValueError("Engine outputs failed reference parity") + report["status"] = "passed" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--inputs", type=Path, required=True, help="Named preprocessed input arrays in NPZ format") + parser.add_argument("--output", type=Path, required=True, help="New directory for engine, inspection, outputs and report") + parser.add_argument("--profiles", type=Path, help="JSON object mapping each input name to min/opt/max shape lists") + parser.add_argument("--fp16", action="store_true", help="Allow FP16 tactics; QDQ quantization is defined by the graph") + parser.add_argument("--tf32", action="store_true") + parser.add_argument("--workspace-mib", type=int, default=1024) + parser.add_argument("--optimization", type=int, choices=range(6), default=3) + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--reference", type=Path, help="Optional NPZ of expected outputs with exact names/shapes/dtypes") + parser.add_argument("--rtol", type=float, default=1e-4) + parser.add_argument("--atol", type=float, default=1e-5) + args = vars(parser.parse_args()) + if args["profiles"] is not None: + args["profiles"] = json.loads(args["profiles"].read_text()) + build(**args) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/tensorrt_deployment.py b/dev/benchmarks/inference/tensorrt_deployment.py new file mode 100644 index 00000000..6c9b809d --- /dev/null +++ b/dev/benchmarks/inference/tensorrt_deployment.py @@ -0,0 +1,247 @@ +"""Compose TensorRT held-out quality, inspection, paired latency and isolated memory.""" + +import json +import os +import subprocess +import sys +from argparse import ArgumentParser +from collections import Counter +from pathlib import Path + +from .inference_pair import run_pair +from .onnx_inference import file_hash + + +def build_bundle(path): + """Bind retained inspection to the exact engine measured by every stage.""" + path = Path(path).resolve(strict=True) + report = json.loads((path / "report.json").read_text()) + engine, layers = path / "model.engine", path / "layers.json" + if report["status"] != "passed" or file_hash(engine) != report["engine"]["sha256"]: + raise ValueError("Build report must describe the exact successful engine") + if file_hash(layers) != report["engine"]["layers_sha256"]: + raise ValueError("Layer inspection differs from the build report") + inspection = json.loads(layers.read_text())["Layers"] + return engine, { + "directory": str(path), + "report_sha256": file_hash(path / "report.json"), + "engine": report["engine"], + "settings": report["settings"], + "weight_types": dict(Counter(layer["Weights"]["Type"] for layer in inspection if "Weights" in layer)), + "gemms": [layer for layer in inspection if layer.get("LayerType") == "gemm"], + } + + +def evaluate( + baseline_build, + candidate_build, + manifest, + inputs, + output, + trials=3, + warmup=10, + repeats=31, + memory_runs=20, + threads=1, + device=0, + pinned=False, + metrics_python=sys.executable, +): + if min(trials, repeats, memory_runs, threads) < 1 or min(warmup, device) < 0: + raise ValueError("Require positive trials/repeats/memory_runs/threads and nonnegative warmup/device") + manifest, inputs = Path(manifest).resolve(strict=True), Path(inputs).resolve(strict=True) + output = Path(output).resolve() + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "manifest": {"path": str(manifest), "sha256": file_hash(manifest)}, + "inputs": {"path": str(inputs), "sha256": file_hash(inputs)}, + "settings": { + "trials": trials, + "warmup": warmup, + "repeats": repeats, + "memory_runs": memory_runs, + "threads": threads, + "device": device, + "pinned_host_io": pinned, + }, + "builds": {}, + "stages": [], + "trials": [], + "scope": ( + "Paired held-out mini_metrics quality, build-bound inspection, adjacent paired host latency and isolated memory. " + "Evaluated is not production acceptance; inspect all five metrics, undefined values and resource trade-offs. " + "Latency excludes preprocessing/loading; device memory is device-wide snapshots, not per-process or transient peaks. " + "Run on quiescent target hardware; no automatic thermal control or disk-cache eviction." + ), + } + + def save(): + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + + def child(name, module, arguments): + command = [sys.executable, "-m", f"dev.benchmarks.inference.{module}", *map(str, arguments), "--output", str(output / name)] + stage = {"name": name, "command": command, "status": "running", "log": f"{name}.log"} + report["stages"].append(stage) + save() + with (output / stage["log"]).open("w") as log: + process = subprocess.run( + command, + cwd=Path(__file__).resolve().parents[3], + env={**os.environ, "PYTHONHASHSEED": "0", "OMP_NUM_THREADS": str(threads)}, + stdout=log, + stderr=subprocess.STDOUT, + check=False, + ) + stage["returncode"] = process.returncode + if process.returncode: + raise RuntimeError(f"{name} failed; see {output / stage['log']}") + result = json.loads((output / name / "report.json").read_text()) + if result["status"] != "passed": + raise RuntimeError(f"Unexpected {name} status: {result['status']}") + stage.update(status="passed", report_sha256=file_hash(output / name / "report.json")) + return result + + def identity(role, engine_info): + if engine_info["sha256"] != report["builds"][role]["engine"]["sha256"]: + raise ValueError(f"{role} engine changed between build and evaluation stages") + + def resource_inputs(result): + if result["inputs"]["sha256"] != report["inputs"]["sha256"]: + raise ValueError("Resource inputs changed between stages") + + try: + report["phase"] = "inspection" + engines = {} + for role, path in (("baseline", baseline_build), ("candidate", candidate_build)): + engines[role], report["builds"][role] = build_bundle(path) + report["phase"] = "quality" + save() + runtime = {"backend": "tensorrt", "device": device} + quality = run_pair(manifest, engines["baseline"], engines["candidate"], output / "quality", runtime, runtime, metrics_python) + report["quality"] = { + "report_sha256": file_hash(output / "quality/report.json"), + "levels": quality["levels"], + "undefined_metrics": quality["undefined_metrics"], + "models": quality["models"], + } + collectors = {} + for role in engines: + result = json.loads((output / f"quality/{role}/report.json").read_text()) + if len(result["model_files"]) != 1: + raise ValueError("Require one TensorRT engine per collector") + identity(role, result["model_files"][0]) + if result["manifest"]["sha256"] != report["manifest"]["sha256"]: + raise ValueError("Quality manifest changed between stages") + collectors[role] = result + if collectors["baseline"]["batches"] != collectors["candidate"]["batches"]: + raise ValueError("Quality input batches differ between engines") + if collectors["baseline"]["runtime"] != collectors["candidate"]["runtime"]: + raise ValueError("Quality runtime environments differ") + report["phase"] = "resources" + save() + flags = ["--device", device, *(["--pinned"] if pinned else [])] + for trial in range(trials): + record = {"trial": trial} + report["trials"].append(record) + latency = child( + f"latency-{trial}", + "tensorrt_pair", + [ + "--baseline", + engines["baseline"], + "--candidate", + engines["candidate"], + "--inputs", + inputs, + "--warmup", + warmup, + "--repeats", + repeats, + *flags, + *(["--reverse"] if trial % 2 else []), + ], + ) + resource_inputs(latency) + for role in engines: + identity(role, latency["models"][role]) + record["latency"] = latency["summary"] + memory = {} + order = ["baseline", "candidate"] if trial % 2 == 0 else ["candidate", "baseline"] + record["memory_order"] = order + for role in order: + result = child( + f"memory-{trial}-{role}", + "tensorrt_memory", + ["--engine", engines[role], "--inputs", inputs, "--runs", memory_runs, "--threads", threads, *flags], + ) + resource_inputs(result) + identity(role, result["engine"]) + memory[role] = result + record.setdefault("memory", {})[role] = result["memory"] + a, b = memory["baseline"], memory["candidate"] + if any(a[key] != b[key] for key in ("settings", "versions", "environment")): + raise ValueError("Memory trial environments or settings differ") + for value in memory.values(): + if any(value["versions"][key] != latency["versions"][key] for key in ("torch", "tensorrt", "numpy")): + raise ValueError("Latency and memory runtime versions differ") + if value["environment"]["gpu"] != latency["environment"]["gpu"]: + raise ValueError("Latency and memory GPU identities differ") + if any(latency["versions"][key] != collectors["baseline"]["runtime"][key] for key in ("tensorrt", "torch")): + raise ValueError("Quality and resource runtime versions differ") + if latency["environment"]["gpu"] != collectors["baseline"]["runtime"]["gpu"]: + raise ValueError("Quality and resource GPU identities differ") + save() + lines = [ + "# TensorRT deployment comparison", + "", + report["scope"], + "", + "| Trial | Paired latency ratio | Baseline warm device bytes | Candidate warm device bytes | " + "Baseline host RSS bytes | Candidate host RSS bytes |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + for trial in report["trials"]: + a, b = [trial["memory"][role]["warm"] for role in ("baseline", "candidate")] + lines.append( + f"| {trial['trial']} | {trial['latency']['median_paired_ratio']:.4f} | " + f"{a['device_used_bytes']} | {b['device_used_bytes']} | " + f"{a['host']['resident_bytes']} | {b['host']['resident_bytes']} |" + ) + lines.extend( + [ + "", + "Latency ratios are candidate / baseline. Device readings include shared/runtime overhead; " + "inspect initialization snapshots in report.json.", + "", + (output / "quality/summary.md").read_text(), + ] + ) + (output / "summary.md").write_text("\n".join(lines) + "\n") + report.update(status="evaluated", phase="complete") + except BaseException as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + for stage in report["stages"]: + if stage["status"] == "running": + stage["status"] = "failed" + raise + finally: + save() + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + for name in ("baseline-build", "candidate-build", "manifest", "inputs", "output"): + parser.add_argument("--" + name, type=Path, required=True) + for name, default in (("trials", 3), ("warmup", 10), ("repeats", 31), ("memory-runs", 20), ("threads", 1), ("device", 0)): + parser.add_argument("--" + name, type=int, default=default) + parser.add_argument("--pinned", action="store_true") + parser.add_argument("--metrics-python", default=sys.executable) + evaluate(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/tensorrt_memory.py b/dev/benchmarks/inference/tensorrt_memory.py new file mode 100644 index 00000000..78faf3ec --- /dev/null +++ b/dev/benchmarks/inference/tensorrt_memory.py @@ -0,0 +1,140 @@ +"""Collect single-engine TensorRT memory snapshots in a fresh Linux process.""" + +import hashlib +import io +import json +import platform +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np + +from .onnx_cpu_memory import resident_memory +from .onnx_inference import file_hash +from .tensorrt_pair import buffers_for + + +def memory_snapshot(torch, device): + """Keep device-wide observations separate from process and allocator counters.""" + torch.cuda.synchronize(device) + free, total = torch.cuda.mem_get_info(device) + return { + "device_used_bytes": total - free, + "device_total_bytes": total, + "torch_allocated_bytes": torch.cuda.memory_allocated(device), + "torch_reserved_bytes": torch.cuda.memory_reserved(device), + "host": resident_memory(), + } + + +def measure(engine, inputs, output, runs=20, threads=1, device=0, pinned=False): + """Call the CLI separately for every engine/trial; no environment installation.""" + if runs < 1 or threads < 1 or device < 0: + raise ValueError("Require positive runs/threads and a nonnegative device") + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "environment": {"platform": platform.platform(), "python": platform.python_version()}, + "settings": {"runs": runs, "threads": threads, "device": device, "pinned_host_io": pinned, "profile": 0}, + "memory": {}, + "messages": [], + "scope": ( + "Run one engine per fresh Linux process. Device used/free is device-wide, not per-process or a transient peak; " + "other activity and WSL accounting can affect attribution. PyTorch counters exclude TensorRT-owned allocations. " + "Host RSS/PSS includes interpreter, imports, inputs, engine and IO; host peak is approximate since exec. " + "No latency or quality acceptance claim. Snapshots precede output serialization." + ), + } + try: + report["memory"]["before_runtime_import"] = {"host": resident_memory()} + try: + import tensorrt as trt + import torch + except ImportError as error: + raise ImportError("Use an explicitly prepared TensorRT and CUDA PyTorch environment; this command installs nothing") from error + torch.set_num_threads(threads) + report["versions"] = {"tensorrt": trt.__version__, "torch": torch.__version__, "numpy": np.__version__} + payload = Path(inputs).read_bytes() + report["inputs"] = {"path": str(inputs), "sha256": hashlib.sha256(payload).hexdigest()} + with np.load(io.BytesIO(payload), allow_pickle=False) as archive: + feeds = {name: archive[name].copy(order="C") for name in archive.files} + del payload + if not feeds or any(not np.isfinite(a).all() for a in feeds.values()): + raise ValueError("Supply finite named input arrays") + + class Logger(trt.ILogger): + def log(self, severity, message): + if severity <= trt.ILogger.WARNING: + report["messages"].append({"severity": str(severity), "message": message}) + + with torch.cuda.device(device): + stream = torch.cuda.Stream(device=device) + report["environment"].update( + gpu=torch.cuda.get_device_name(device), compute_capability=list(torch.cuda.get_device_capability(device)) + ) + report["memory"]["cuda_initialized"] = memory_snapshot(torch, device) + logger = Logger() + trt.init_libnvinfer_plugins(logger, "") + runtime = trt.Runtime(logger) + serialized = Path(engine).read_bytes() + report["engine"] = {"path": str(engine), "sha256": hashlib.sha256(serialized).hexdigest(), "bytes": len(serialized)} + model = runtime.deserialize_cuda_engine(serialized) + del serialized + if model is None: + raise RuntimeError("Could not deserialize engine") + report["memory"]["engine_loaded"] = memory_snapshot(torch, device) + context = model.create_execution_context() + if context is None: + raise RuntimeError("Could not create execution context") + with torch.cuda.stream(stream): + buffers, input_names = buffers_for(model, context, feeds, torch, trt, device, pinned) + report["io"] = { + name: {"shape": list(host.shape), "dtype": str(host.numpy().dtype), "input": name in input_names} + for name, (_, host) in buffers.items() + } + if all(item["input"] for item in report["io"].values()): + raise ValueError("Require at least one engine output") + report["context_memory_bytes"] = model.device_memory_size_v2 + report["memory"]["context_and_io"] = memory_snapshot(torch, device) + for _ in range(runs): + with torch.cuda.stream(stream): + for name in input_names: + gpu, host = buffers[name] + gpu.copy_(host, non_blocking=pinned) + if not context.execute_async_v3(stream.cuda_stream): + raise RuntimeError("Engine execution failed") + for name, (gpu, host) in buffers.items(): + if name not in input_names: + host.copy_(gpu, non_blocking=pinned) + stream.synchronize() + if any(not np.isfinite(host.numpy()).all() for name, (_, host) in buffers.items() if name not in input_names): + raise ValueError("Nonfinite engine outputs") + report["memory"]["warm"] = memory_snapshot(torch, device) + arrays = {name: host.numpy() for name, (_, host) in buffers.items() if name not in input_names} + np.savez(output / "outputs.npz", **arrays) + report["outputs_sha256"] = file_hash(output / "outputs.npz") + report["status"] = "passed" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + for name in ("engine", "inputs", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--runs", type=int, default=20) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--pinned", action="store_true") + measure(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/inference/tensorrt_pair.py b/dev/benchmarks/inference/tensorrt_pair.py new file mode 100644 index 00000000..7748c612 --- /dev/null +++ b/dev/benchmarks/inference/tensorrt_pair.py @@ -0,0 +1,213 @@ +"""Measure paired TensorRT engine latency with explicit inputs and retained trials.""" + +import hashlib +import io +import json +import math +import platform +import statistics +import time +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np + +from .onnx_inference import file_hash + + +def paired_trials(execute, warmup, repeats, reverse=False): + """Alternate adjacent measurements; keep each ratio paired within its trial.""" + if warmup < 0 or repeats < 1: + raise ValueError("Require nonnegative warmup and positive repeats") + for index in range(warmup + repeats): + order = ["baseline", "candidate"] + if bool(index % 2) ^ reverse: + order.reverse() + seconds = {name: execute(name) for name in order} + if any(not math.isfinite(value) or value <= 0 for value in seconds.values()): + raise ValueError("Measured durations must be finite and positive") + if index >= warmup: + yield {"index": index - warmup, "order": order, "seconds": seconds} + + +def summarize(trials): + ratios = [trial["seconds"]["candidate"] / trial["seconds"]["baseline"] for trial in trials] + return { + "median_seconds": {name: statistics.median(trial["seconds"][name] for trial in trials) for name in ("baseline", "candidate")}, + "candidate_over_baseline_ratios": ratios, + "median_paired_ratio": statistics.median(ratios), + } + + +def buffers_for(engine, context, feeds, torch, trt, device, pinned): + names = [engine.get_tensor_name(i) for i in range(engine.num_io_tensors)] + inputs = {name for name in names if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT} + if inputs != set(feeds): + raise ValueError("Input names do not match engine") + for name in names: + if engine.get_tensor_format(name) != trt.TensorFormat.LINEAR or engine.get_tensor_location(name) != trt.TensorLocation.DEVICE: + raise ValueError(f"Require linear device IO: {name}") + for name, value in feeds.items(): + if engine.is_shape_inference_io(name): + raise ValueError(f"Shape-tensor inputs are not supported by this probe: {name}") + if np.dtype(trt.nptype(engine.get_tensor_dtype(name))) != value.dtype: + raise ValueError(f"Input dtype does not match engine: {name}") + if not context.set_input_shape(name, value.shape): + raise ValueError(f"Input shape is outside engine profile 0: {name}") + unresolved = context.infer_shapes() + if unresolved: + raise ValueError(f"Unresolved engine shapes: {unresolved}") + buffers = {} + for name in names: + shape = tuple(context.get_tensor_shape(name)) + if any(n <= 0 for n in shape): + raise ValueError(f"Unresolved or empty tensor: {name}") + dtype = torch.from_numpy(np.empty((), dtype=trt.nptype(engine.get_tensor_dtype(name)))).dtype + host = torch.empty(shape, dtype=dtype, pin_memory=pinned) + if name in inputs: + host.copy_(torch.from_numpy(feeds[name])) + gpu = torch.empty(shape, dtype=dtype, device=f"cuda:{device}") + if not context.set_tensor_address(name, gpu.data_ptr()): + raise RuntimeError(f"TensorRT rejected IO address: {name}") + buffers[name] = (gpu, host) + return buffers, tuple(sorted(inputs)) + + +def benchmark(baseline, candidate, inputs, output, warmup=10, repeats=31, reverse=False, device=0, pinned=False): + if warmup < 0 or repeats < 1 or device < 0: + raise ValueError("Require nonnegative warmup/device and positive repeats") + payload = Path(inputs).read_bytes() + input_hash = hashlib.sha256(payload).hexdigest() + with np.load(io.BytesIO(payload), allow_pickle=False) as data: + feeds = {name: data[name].copy(order="C") for name in data.files} + del payload + if not feeds or any(not np.isfinite(a).all() for a in feeds.values()): + raise ValueError("Supply finite named input arrays") + try: + import tensorrt as trt + import torch + except ImportError as error: + raise ImportError("Use an explicitly prepared TensorRT and CUDA PyTorch environment; this command installs nothing") from error + + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": file_hash(__file__), + "versions": {"tensorrt": trt.__version__, "torch": torch.__version__, "numpy": np.__version__}, + "environment": {"platform": platform.platform(), "python": platform.python_version()}, + "settings": {"warmup": warmup, "repeats": repeats, "reverse": reverse, "device": device, "pinned_host_io": pinned, "profile": 0}, + "inputs": {"path": str(inputs), "sha256": input_hash}, + "models": {}, + "messages": [], + "scope": ( + "Synchronized host latency: preallocated host-to-device inputs, engine execution and device-to-host outputs. " + "Excludes decoding/preprocessing, allocation, deserialization and shape changes. Both engines/contexts coexist. " + "No CUDA-event timing or CUDA graph capture. Not a quality, integer-placement or total-memory acceptance test." + ), + } + + class Logger(trt.ILogger): + def log(self, severity, message): + if severity <= trt.ILogger.WARNING: + report["messages"].append({"severity": str(severity), "message": message}) + + try: + with torch.cuda.device(device): + report["environment"].update( + gpu=torch.cuda.get_device_name(device), compute_capability=list(torch.cuda.get_device_capability(device)) + ) + report["environment"].update(torch_threads=torch.get_num_threads(), torch_interop_threads=torch.get_num_interop_threads()) + logger = Logger() + trt.init_libnvinfer_plugins(logger, "") + runtime = trt.Runtime(logger) + stream = torch.cuda.Stream(device=device) + models = {} + for name, path in (("baseline", baseline), ("candidate", candidate)): + before = time.perf_counter() + serialized = Path(path).read_bytes() + read_seconds = time.perf_counter() - before + info = { + "path": str(path), + "sha256": hashlib.sha256(serialized).hexdigest(), + "bytes": len(serialized), + "read_seconds": read_seconds, + } + report["models"][name] = info + torch.cuda.synchronize(device) + before = time.perf_counter() + engine = runtime.deserialize_cuda_engine(serialized) + if engine is None: + raise RuntimeError(f"Could not deserialize {name} engine") + context = engine.create_execution_context() + if context is None: + raise RuntimeError(f"Could not create {name} execution context") + torch.cuda.synchronize(device) + info["deserialize_and_context_seconds"] = time.perf_counter() - before + del serialized + with torch.cuda.stream(stream): + buffers, input_names = buffers_for(engine, context, feeds, torch, trt, device, pinned) + info["context_memory_bytes"] = engine.device_memory_size_v2 + info["io"] = { + key: {"shape": list(host.shape), "dtype": str(host.numpy().dtype), "input": key in input_names} + for key, (_, host) in buffers.items() + } + models[name] = engine, context, buffers, input_names + # Outputs can use different precisions but must describe the same named shapes. + contracts = [ + {key: value["shape"] for key, value in info["io"].items() if not value["input"]} for info in report["models"].values() + ] + if not contracts[0] or contracts[0] != contracts[1]: + raise ValueError("Engine output names/shapes must match") + stream.synchronize() + + def execute(name): + _, context, buffers, input_names = models[name] + with torch.cuda.stream(stream): + before = time.perf_counter() + for key in input_names: + gpu, host = buffers[key] + gpu.copy_(host, non_blocking=pinned) + if not context.execute_async_v3(stream.cuda_stream): + raise RuntimeError(f"Execution failed: {name}") + for key, (gpu, host) in buffers.items(): + if key not in input_names: + host.copy_(gpu, non_blocking=pinned) + stream.synchronize() + return time.perf_counter() - before + + report["trials"] = [] + report["trials"].extend(paired_trials(execute, warmup, repeats, reverse)) + report["summary"] = summarize(report["trials"]) + for name, (_, _, buffers, input_names) in models.items(): + arrays = {key: host.numpy() for key, (_, host) in buffers.items() if key not in input_names} + np.savez(output / f"{name}-outputs.npz", **arrays) + report["models"][name]["outputs_sha256"] = file_hash(output / f"{name}-outputs.npz") + if any(not np.isfinite(a).all() for a in arrays.values()): + raise ValueError(f"Nonfinite final outputs: {name}") + report["status"] = "passed" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--inputs", type=Path, required=True, help="Named preprocessed NPZ inputs with exact shapes and dtypes") + parser.add_argument("--output", type=Path, required=True, help="New report/output directory") + parser.add_argument("--warmup", type=int, default=10) + parser.add_argument("--repeats", type=int, default=31) + parser.add_argument("--reverse", action="store_true", help="Reverse the alternating execution order") + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--pinned", action="store_true", help="Use preallocated pinned host IO and nonblocking copies") + benchmark(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/models.py b/dev/benchmarks/models.py index 1a62384e..2f0a8903 100644 --- a/dev/benchmarks/models.py +++ b/dev/benchmarks/models.py @@ -46,3 +46,31 @@ def __init__(self): def forward(self, images): return self.fc(self.features(images)) + + +class DenseImageMLP(torch.nn.Module): + """Spatial image MLP for measuring QT when Linear dominates the backbone. + + This intentionally dense architecture is a compute profile, not a proposed + replacement for CNNs or a model-quality baseline. Its fixed input pooling + keeps construction and checkpoint restoration independent of dataset size. + """ + + default_transform = Compose([torch.nn.Identity()]) + + def __init__(self): + super().__init__() + self.features = torch.nn.Sequential( + torch.nn.AdaptiveAvgPool2d((28, 28)), + torch.nn.Flatten(), + torch.nn.Linear(3 * 28 * 28, 2048), + torch.nn.ReLU(), + torch.nn.Linear(2048, 2048), + torch.nn.ReLU(), + torch.nn.Linear(2048, 2048), + torch.nn.ReLU(), + ) + self.fc = torch.nn.Linear(2048, 10) + + def forward(self, images): + return self.fc(self.features(images)) diff --git a/dev/benchmarks/prepare_inputs.py b/dev/benchmarks/prepare_inputs.py new file mode 100644 index 00000000..23e0df34 --- /dev/null +++ b/dev/benchmarks/prepare_inputs.py @@ -0,0 +1,8 @@ +"""Compatibility factory path retained by historical preprocessing recipes. + +Run new preparation commands through dev.benchmarks.inference.prepare_inputs. +""" + +from dev.benchmarks.inference.prepare_inputs import repository_preprocess + +__all__ = ["repository_preprocess"] diff --git a/dev/benchmarks/reporting.md b/dev/benchmarks/reporting.md new file mode 100644 index 00000000..07d9302b --- /dev/null +++ b/dev/benchmarks/reporting.md @@ -0,0 +1,108 @@ +# Continuous deployment reporting + +The maintained path is target execution → compact history → optional release +storage and Pages. Local adapters are tested; target runners, authenticated storage +and public rendering still need live qualification. + +## Opt-in target GPU workflow + +```bash +export BENCHMARK_PYTHON=/path/to/prepared-tensorrt-env/bin/python +export BENCHMARK_METRICS_PYTHON=/path/to/metrics-env/bin/python +export TRT_BASELINE_MODEL=/path/to/float/model.onnx +export TRT_CANDIDATE_MODEL=/path/to/calibrated/model.onnx +export TRT_INFERENCE_MANIFEST=/path/to/heldout/manifest.json +export TRT_INPUTS=/path/to/heldout/batch-00000.npz +export TRT_PROFILES=/path/to/profiles.json +CUDA_VISIBLE_DEVICES=0 bash dev/check-tensorrt-deployment.sh fresh-results +``` + +The harness checks prepared runtimes, rebuilds both engines and composes +[inference evaluation](inference.md). It installs nothing. Keep external weights +beside ONNX models; source models must match the manifest's class/score contract. +Both builds allow FP16 and disable TF32, with optimization level 1 and 1 GiB +workspace by default. Override with `TRT_OPTIMIZATION` and `TRT_WORKSPACE_MIB`. + +`.github/workflows/tensorrt-deployment.yml` exposes the same harness on a self-hosted +Linux GPU runner. Keep prepared environments/artifacts outside the checkout. + +| Repository variable | Purpose | +| --- | --- | +| `TRT_BENCHMARK_PYTHON` | Maps to the harness's `BENCHMARK_PYTHON` | +| `BENCHMARK_METRICS_PYTHON` | Prepared metrics interpreter | +| `TRT_BASELINE_MODEL`, `TRT_CANDIDATE_MODEL` | Absolute source ONNX paths | +| `TRT_INFERENCE_MANIFEST`, `TRT_INPUTS`, `TRT_PROFILES` | Absolute input/profile paths | +| `TRT_RUNNER_LABEL`, `TRT_CUDA_VISIBLE_DEVICES` | Defaults `gpu`, `0` | +| `TRT_THREADS`, `TRT_OPTIMIZATION`, `TRT_WORKSPACE_MIB` | Optional resource/build settings | +| `ENABLE_TENSORRT_BENCHMARKS` | Exactly `true` enables scheduled execution | +| `TRT_REPORT_PROFILE`, `TRT_REPORT_NOTE` | Public workload/measurement description | +| `TRT_PERFORMANCE_VALID` | Exactly `true` opts into resource comparisons; default false | + +Raw and compact artifacts include run ID/attempt and expire after 90 days. +Failures retain phase, exit code and available logs. Workflow concurrency does +not coordinate unrelated jobs: use an idle allocation for performance evidence. + +## Compact report history and local dashboard + +```bash +.venv/bin/python -m dev.benchmarks.reporting.report_history archive \ + --report fresh-results/evaluation/report.json --history local-history \ + --run-id run-123-attempt-1 --revision FULL_SOURCE_COMMIT_HASH \ + --profile 'Blair flat / target GPU / batch 8' --note 'Measurement conditions' +.venv/bin/python -m dev.benchmarks.reporting.report_history render --history local-history +``` + +Open `local-history/index.html`. CPU composed reports use the same command with +`--report cpu-comparison/report.json`. Keep all referenced child reports in place: +archival verifies hashes, metric deltas, identities and measurement settings. +New CPU reports enforce requested trial counts; older reports visibly retain an +unknown requested count. CPU medians/RSS and GPU paired timing/snapshots remain +distinct. Missing/undefined values and failed runs remain visible. + +Run IDs are immutable: identical retries succeed, conflicting content fails. +`records/` is the durable input; HTML is regenerated. Records omit predictions, +local artifact paths and exception text but include explicitly supplied public +metadata. Review that metadata before hosting. `--performance-valid` is a publisher +declaration about measurement conditions, not automatic qualification; failed runs +never receive eligible resource comparisons. + +For preflight/build failure, archive the nonzero root `status.json`. A successful +root status cannot replace detailed evaluation. The shared wrapper +`bash dev/check-report-history.sh RESULTS HISTORY` selects the proper source and +requires `BENCHMARK_RUN_ID`, `BENCHMARK_REVISION`, `BENCHMARK_PROFILE`; optional +variables are `BENCHMARK_RUN_URL`, `BENCHMARK_NOTE`, `BENCHMARK_PERFORMANCE_VALID`. +It uses `BENCHMARK_PYTHON` and needs only Python's standard library. + +## Draft release storage client + +```bash +.venv/bin/python -m dev.benchmarks.reporting.release_history \ + --repository OWNER/REPO --records local-history/records --output fresh-preview +``` + +Add `--upload` to append records with readback verification; omit `--records` to +restore only. Requires authenticated `gh` for github.com, with contents write +permission for uploads. Supply tokens through `GH_TOKEN`, never command arguments. +Use a fresh output directory on every attempt. + +Records are stored in `benchmark-history-YYYY-MM` draft/prereleases by UTC creation +month. Published archives are readable but not appendable. The client validates +merged history before writing and never overwrites/deletes assets. Partial uploads +can be retried with identical inputs. Draft assets remain mutable by maintainers; +keep an independent backup. These tags do not match the PyPI workflow's `v*` trigger. + +## Opt-in public history publisher + +After review and merge to the default branch: + +1. Configure the target runner and variables above. +2. Set GitHub Pages publishing to GitHub Actions and enable `ENABLE_BENCHMARK_HISTORY=true`. +3. Run a real success, failure and publisher-only retry. Verify stored asset readback, + visible metrics/resource scopes, failure visibility and the public page. + +The separate GitHub-hosted publisher downloads only the compact artifact by ID, +restores/appends monthly history and deploys HTML/JSON. It requires contents/pages +write and id-token permissions, and runs only on the default branch. Publishing +is serialized; retry artifacts include the attempt number. No live activation has +been performed. CPU-producing workflows and training-history adapters remain open; +see [the roadmap](../../docs/quantization-roadmap.md). diff --git a/dev/benchmarks/reporting/__init__.py b/dev/benchmarks/reporting/__init__.py new file mode 100644 index 00000000..8179a128 --- /dev/null +++ b/dev/benchmarks/reporting/__init__.py @@ -0,0 +1 @@ +"""Development benchmarks: reporting.""" diff --git a/dev/benchmarks/reporting/confusion.md b/dev/benchmarks/reporting/confusion.md new file mode 100644 index 00000000..645c2212 --- /dev/null +++ b/dev/benchmarks/reporting/confusion.md @@ -0,0 +1,122 @@ +# Whole-matrix confusion diagnostics + +Confusion figures retain every model output index on both axes, including +prediction-only and unobserved classes, in the same order across epochs. Rows +are true classes and columns are predicted classes. Class names can be resolved +from the saved model/config classifier metadata. No taxonomy regrouping or +selected-pair view replaces the full matrix. + +The specialized reporting logic lives in `mini_trainer/logging/confusion.py`: +rank reduction, whole-matrix overviews, palette encoding and numerical artifacts. +The existing `visualization/plot.py` retains generic heatmap/colorbar rendering; +its byte-color mapping now uses bounded chunks with regression checks against +the previous RGB pixels. No new runtime dependencies or background service are +introduced. + +## Dashboard images and local detail + +TensorBoard/W&B receive RGB previews with a matrix side of at most 1,536 pixels, +plus a 200-pixel legend and a 24-pixel caption. Each preview covers the entire +matrix. For N classes its block width is `ceil(N / 1536)`; a preview cell is the +arithmetic mean of the row-normalized probabilities in that block. Partial edge +blocks use their actual cell counts. This is an overview of probabilities, not +an aggregated or renormalized confusion matrix. The caption, `/overview_mean` +tag and metadata explicitly identify the reduction. Small images enlarge with +nearest-neighbor sampling; there are no cell borders or spatial smoothing. +Isolated errors may become less visible in the mean overview; native-resolution +images and numerical matrices retain them. + +Soft images use 128 positive logarithmic color levels over probabilities +`[1e-6, 1]`, plus dedicated black zero and magenta invalid/negative colors. Values +below the positive floor use the lowest positive color, never the zero color. +Hard images retain the 256-color magma mapping on the same fixed scale. Scales +remain comparable across epochs. Posterization changes displayed colors only; +raw values are retained. The full soft PNG is indexed (palette mode), without +dithering. The matching discrete colorbar is saved separately, so its text does +not expand the matrix palette. Dashboard previews include the legend. + +Each epoch saves the following under +`model/logs/figures/epoch-NNNN/Confusion_matrix_lvlL/` and +`Soft_confusion_matrix_lvlL/`: + +- `matrix.png`: one pixel per original class pair, without the old 5,000-class + maximum-pooling limit. The separate colorbar does not change matrix dimensions. +- `counts.npz` for hard matrices: exact integer COO arrays `rows`, `columns`, + `counts`, plus `shape`. Absent entries are zero. +- `probability_sums.npy` for soft matrices: the original accumulated float32 + probability sums, before normalization, clipping or posterization. +- `row_support.npy`: hard true-label counts or the soft row probability mass + used for normalization. Empty rows remain zero. +- `metadata.json`: original class indices, orientation, normalization, palette, + invalid-cell count and preview block shape; and `colorbar.png`. + +The soft NPY is deliberately uncompressed to avoid spending logging time +compressing high-entropy floats. It is about 100 MB for 5,000 classes and 400 MB +for 10,000 classes per saved epoch/level. Local storage therefore increases even +though dashboard traffic decreases. NumPy can inspect it with `mmap_mode='r'`. +Storage throughput on the production filesystem needs qualification. Accumulation +and exact matrix storage still scale quadratically with class count. Full-size +PNGs remain large decoded images; the bounded preview is what protects dashboard +responsiveness. This increment does not add a tile server or custom viewer. + +## Distributed reporting + +Every rank participates in confusion collection. Small shape descriptors allow +even a rank with no validation samples to participate; count and probability +matrices are summed onto rank zero in bounded chunks. NCCL uses temporary CUDA +chunks rather than placing the entire matrix on the GPU. Only rank zero renders, +writes artifacts and forwards preview images. Soft accumulation buffers are not +mutated, so repeated reporting does not double counts. Counts include validation +sampler padding, matching the samples actually reported; sample-ID deduplication +is outside this change. + +CPU two-process tests cover an empty rank, prediction-only classes and repeated +collection. CUDA/NCCL and live TensorBoard/W&B uploads remain target-job checks. + +## Reproduce rendering measurements + +The benchmark uses seeded dense soft probabilities with 20 large diagonal blocks +and a diagonal signal. Timing includes normalization, image rendering and file +writes. The new path also writes the exact matrix and full-resolution PNG, which +the old path did not retain. RSS includes imports and the input matrix; it is not +training's peak memory. Run each case in a fresh process: + +```bash +git show d521cef:mini_trainer/visualization/plot.py > /tmp/heatmap-before.py +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MPLBACKEND=Agg \ + .venv/bin/python -m dev.benchmarks.reporting.confusion --classes 5000 \ + --baseline /tmp/heatmap-before.py --output /tmp/confusion-before +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MPLBACKEND=Agg \ + .venv/bin/python -m dev.benchmarks.reporting.confusion --classes 5000 \ + --output /tmp/confusion-after +# Repeat with --classes 10000 and new output directories. +``` + +Representative local CPU results (decimal MB for file sizes): + +| Classes | Old/new seconds | Old/new peak RSS MiB | Old/new dashboard PNG MB | +| --- | ---: | ---: | ---: | +| 5,000 | 9.58 / 2.77 | 2,994 / 907 | 47.18 / 1.34 | +| 10,000 | 12.12 / 6.62 | 3,236 / 1,258 | 33.00 / 1.11 | + +The new full-resolution soft PNGs are 15.78 MB and 62.67 MB respectively. The old +10,000-class dashboard was already pooled to 5,000 cells per side; the new local +PNG preserves all 10,000. These are synthetic, host-specific measurements, +excluding distributed reduction and remote dashboard upload. + +The existing browser benchmark also accepts PNGs. With the disposable browser +environment described in [dendrogram.md](dendrogram.md), run: + +```bash +/tmp/svg-browser-env/bin/python dev/benchmarks/reporting/svg_browser.py \ + /tmp/confusion-before/dashboard.png /tmp/confusion-after/dashboard.png \ + --output /tmp/confusion-browser +``` + +For the final 5,000-class dashboard PNGs, headless Chromium's median decode plus +forced raster time (three interleaved runs per image/size, fresh contexts) was +1,132 ms before versus 52.6 ms after at a 1,200-pixel display, and 1,242 ms versus +170 ms at 4,800 pixels. This compares the old large dashboard image with the new +captioned whole-matrix preview; it does not imply that the full-resolution local +PNG is cheap to display. Timings exclude Python/browser startup, network transfer +and PNG encoding. Live TensorBoard/W&B end-to-end performance was not measured. diff --git a/dev/benchmarks/reporting/confusion.py b/dev/benchmarks/reporting/confusion.py new file mode 100644 index 00000000..14c5e00b --- /dev/null +++ b/dev/benchmarks/reporting/confusion.py @@ -0,0 +1,57 @@ +"""Fresh-process dense soft-confusion rendering benchmark, including artifact I/O.""" + +import argparse +import importlib.util +import json +import resource +import time +from pathlib import Path + +import numpy as np +from matplotlib import pyplot as plt + +from mini_trainer.logging.confusion import confusion_report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--classes", type=int, default=5000) + parser.add_argument("--baseline", type=Path, help="Previous visualization/plot.py saved from Git") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + n = args.classes + rng = np.random.default_rng(42) + matrix = np.empty((n, n), dtype=np.float32) + columns = np.arange(n) + for start in range(0, n, 64): + rows = np.arange(start, min(n, start + 64)) + values = rng.random((len(rows), n), dtype=np.float32) * 0.01 + values += (rows[:, None] // max(1, n // 20) == columns[None, :] // max(1, n // 20)) * 0.2 + values[np.arange(len(rows)), rows] += 5 + matrix[rows] = values / values.sum(axis=1, keepdims=True) + if args.baseline: + spec = importlib.util.spec_from_file_location("heatmap_baseline", args.baseline) + baseline = importlib.util.module_from_spec(spec) + spec.loader.exec_module(baseline) + started = time.perf_counter() + if args.baseline: + preview = baseline.plot_heatmap(matrix) + else: + preview = confusion_report(matrix, soft=True, directory=args.output) + plt.imsave(args.output / "dashboard.png", preview) + result = { + "classes": n, + "baseline": str(args.baseline) if args.baseline else None, + "seconds": time.perf_counter() - started, + "peak_rss_mib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, + "dashboard_shape": list(preview.shape), + "artifact_bytes": {p.name: p.stat().st_size for p in args.output.iterdir()}, + "scope": "CPU rendering and PNG/data export; RSS includes imports/input; excludes DDP and remote upload", + } + (args.output / "measurement.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/reporting/dendrogram.md b/dev/benchmarks/reporting/dendrogram.md new file mode 100644 index 00000000..bb67bd18 --- /dev/null +++ b/dev/benchmarks/reporting/dendrogram.md @@ -0,0 +1,146 @@ +# Dendrogram scaling and artifact validation + +The renderer uses an iterative SciPy-linkage layout, retaining the original +right-child-first leaf order, distance-based radial positions and flat-cluster +assignments. It does not construct recursive BioPython/pyCirclize tree objects. +Same-color branches share paths; adjacent same-color taxonomy bands are merged. +Circular arcs use bounded chords or cubic Bezier segments instead of dense sampled +polylines. +Every leaf label remains present, including repeated display names. Single-class +levels render without attempting linkage. Taxonomy colors now follow stable +first-occurrence order rather than unordered set iteration. + +Training saves dendrogram SVGs with text elements instead of glyph outlines, +and retains all figure types under `model/logs/figures/epoch-NNNN/` even with the +default metrics-only logger. Matrix images are PNGs; dendrograms remain vector +SVGs. Rank zero owns local artifact writes. Existing external logging backends +still receive the figures. A failed export closes all created dendrogram figures. +Font appearance can depend on the SVG viewer's installed fonts. + +Species-name resolution uses at most eight concurrent lookups, reuses resolved +class lists across epochs (a bounded process-local cache), and falls back to the +original labels when the service or its disk cache is unavailable. GBIF HTTP +requests now have a ten-second socket timeout. This is not a hard total deadline +for taxonomy preparation. A cold large taxonomy still requires real API requests; +the subsequent epochs reuse the result. Logs distinguish label resolution, +per-level rendering and final export time. Pairwise model distances and Ward +linkage remain quadratic in class count; this change does not establish full +production-taxonomy memory capacity or multi-GPU figure latency. + +## Reproduce locally + +Use the existing environment with the recommended plotting dependencies. Each +invocation is a fresh process and includes linkage, layout, render and SVG save. +Input distances are seeded synthetic values; model distance computation and +network name lookup are excluded. Both baseline and current renderer use SVG +text, so the comparison isolates layout/path changes. + +```bash +git show 863a85c:mini_trainer/visualization/dendrogram.py > /tmp/dendrogram-before.py +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MPLBACKEND=Agg \ + .venv/bin/python -m dev.benchmarks.reporting.dendrogram --classes 3422 \ + --module-file /tmp/dendrogram-before.py --output /tmp/dendrogram-baseline +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MPLBACKEND=Agg \ + .venv/bin/python -m dev.benchmarks.reporting.dendrogram --classes 3422 \ + --output /tmp/dendrogram-current +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MPLBACKEND=Agg \ + .venv/bin/python -m dev.benchmarks.reporting.dendrogram --classes 1100 --deep \ + --output /tmp/dendrogram-deep +``` + +Representative local CPU measurements, Python 3.13, Matplotlib 3.10.9, +BioPython 1.87 and pyCirclize 1.10.1, from the same seeded workload: + +| 3,422 classes | Previous renderer | New renderer | +| --- | ---: | ---: | +| Layout/render, including linkage | 18.38 s | 2.78 s | +| Including SVG export | 24.53 s | 6.82 s | +| SVG bytes | 4,309,322 | 2,816,344 | +| SVG path elements | 20,529 | 27 | +| Leaf labels | 3,422 | 3,422 | + +In that first increment, path coordinates occupy 1,492,596 bytes (53%) and the 3,422 +text groups occupy 1,305,053 bytes (46%). Batching removes path-element overhead, +but retains the branch geometry. Matplotlib repeats the font-family list, style +and position/rotation attributes for each label. Shared text styles and lower +coordinate precision were not yet applied in that measurement; see the next +section for the subsequent optimization. +Gzip compresses this example to 769,175 bytes without changing its contents. + +The 1,100-leaf comb tree previously raised `RecursionError` during branch +coloring. The new renderer exported all leaves with seven SVG paths in about +1.9 seconds without changing Python's recursion limit. These are host-specific +single-process measurements, not production training performance promises. + +Regression checks cover deep trees, singleton levels, duplicate names, linkage +geometry and cluster identity, label-cache reuse and outages, SVG text retention, +figure cleanup after failure, and local saving without TensorBoard/W&B. + +## Bounded simplification and compact SVG export + +The next increment replaces short circular arcs with chords only when the +sagitta is at most 0.02 points in figure space (using the full figure width as a +conservative scale bound). This is at most 0.027 CSS pixels at native size, or +0.11 pixels at 4x zoom. Larger arcs retain cubic curves, now at most 45 degrees +per segment; their radial approximation error is below 0.006 points at the +maximum figure size. Branch endpoints, leaf order, cluster assignments and all +labels remain intact. Narrow taxonomy bands consequently become quadrilaterals. +Backend path simplification is disabled on the branch paths to keep it from +adding a second uncontrolled approximation. Magnification far beyond the design +scale can reveal these approximations; this is not exact analytic geometry. + +`save_dendrogram_svg` rounds path coordinates to 0.001 points (at most 0.00071 +points Euclidean rounding error), deduplicates identical text styles, and removes +empty per-label group wrappers. Label strings, fonts, colors, positions and +rotations remain unchanged. Ordinary returned Matplotlib figures benefit from +simpler geometry; training's SVG export additionally applies the markup changes. + +Like-for-like local CPU results for the same 3,422-class input, relative to the +iterative renderer introduced in `909cb09`: + +| Measurement | Before simplification | After simplification | +| --- | ---: | ---: | +| Geometry vertices, including curve controls and bands | 63,719 | 28,980 | +| Figure construction, including linkage | 2.70 s | 1.03 s | +| Construction and SVG export | 6.48 s | 3.22 s | +| SVG bytes | 2,816,344 | 1,041,576 | +| SVG paths / leaf labels | 27 / 3,422 | 27 / 3,422 | +| Chromium decode + raster, 1,200 px square | 329 ms | 252 ms | +| Chromium decode + raster, 4,800 px square | 421 ms | 357 ms | + +Browser values are medians of five interleaved before/after measurements, each +in a fresh browser context, using headless Chromium 151. They include SVG image +decode, canvas drawing and forced pixel readback, excluding process startup, +network transfer and PNG encoding. They do not measure interactive panning or +production dashboard integration. Rasterization alone improved more modestly +(189 to 177 ms at 1,200 px; 311 to 285 ms at 4,800 px); labels still cost work. +These timings are host-specific, not GPU training guarantees. + +Reproduce the figure benchmark with the prior layout and the current exporter: + +```bash +git show 909cb09:mini_trainer/visualization/_dendrogram_layout.py > /tmp/layout-before.py +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MPLBACKEND=Agg \ + .venv/bin/python -m dev.benchmarks.reporting.dendrogram --classes 3422 \ + --layout-file /tmp/layout-before.py --output /tmp/svg-before +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MPLBACKEND=Agg \ + .venv/bin/python -m dev.benchmarks.reporting.dendrogram --classes 3422 \ + --output /tmp/svg-after +``` + +Browser verification uses a disposable environment, without adding dependencies +to training. The benchmark saves raster images and all timing samples: + +```bash +uv venv /tmp/svg-browser-env +uv pip install --python /tmp/svg-browser-env/bin/python playwright +/tmp/svg-browser-env/bin/python -m playwright install chromium +/tmp/svg-browser-env/bin/python dev/benchmarks/reporting/svg_browser.py \ + /tmp/svg-before/dendrogram.svg /tmp/svg-after/dendrogram.svg \ + --output /tmp/svg-browser-results +``` + +Regression tests check chord deviation, cubic radial error, unchanged endpoints, +text/style/transform retention, and export idempotence, alongside the existing +large/deep-tree and logging checks. Pixel comparisons complement those geometric +checks; they cannot establish visibility at arbitrary zoom levels. diff --git a/dev/benchmarks/reporting/dendrogram.py b/dev/benchmarks/reporting/dendrogram.py new file mode 100644 index 00000000..04689813 --- /dev/null +++ b/dev/benchmarks/reporting/dendrogram.py @@ -0,0 +1,86 @@ +"""Fresh-process CPU benchmark for complete dendrogram rendering and SVG export.""" + +import argparse +import importlib.util +import json +import time +from pathlib import Path +from unittest.mock import patch + +import numpy as np +from matplotlib import pyplot as plt +from matplotlib import rc_context +from scipy.spatial.distance import pdist, squareform + +import mini_trainer.visualization.dendrogram as current +from mini_trainer.visualization import save_dendrogram_svg + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--classes", type=int, default=3422) + parser.add_argument("--deep", action="store_true") + baseline = parser.add_mutually_exclusive_group() + baseline.add_argument("--module-file", type=Path, help="Optional baseline dendrogram.py saved from Git") + baseline.add_argument("--layout-file", type=Path, help="Optional baseline _dendrogram_layout.py saved from Git") + parser.add_argument("--output", type=Path, required=True, help="New directory for SVG and measurement JSON") + args = parser.parse_args() + if args.classes < 2: + parser.error("--classes must be at least 2") + module = current + if args.module_file: + spec = importlib.util.spec_from_file_location("dendrogram_baseline", args.module_file) + if spec is None or spec.loader is None: + raise ValueError("Cannot load baseline module") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + renderer = module.render_linkage if hasattr(module, "render_linkage") else None + if args.layout_file: + spec = importlib.util.spec_from_file_location("layout_baseline", args.layout_file) + layout = importlib.util.module_from_spec(spec) + spec.loader.exec_module(layout) + renderer = layout.render_linkage + n = args.classes + distances = squareform(pdist(np.random.default_rng(42).normal(size=(n, 8)))) + linkage = module.linkage + if args.deep: + z = np.array([[0, 1, 1.0, 2]] + [[n + i - 1, i + 1, 1.0 + i / n, i + 2] for i in range(1, n - 1)]) + + def linkage(*args, **kwargs): + return z + + args.output.mkdir(parents=True, exist_ok=False) + started = time.perf_counter() + with patch.object(module, "linkage", linkage), patch.object(module, "render_linkage", renderer, create=True): + fig, info = module._plot_probabilistic_dendrogram( + distances, [f"Species {i}" for i in range(n)], apriori=[f"Group {i % 5}" for i in range(n)] + ) + rendered = time.perf_counter() + path = args.output / "dendrogram.svg" + try: + if args.module_file or args.layout_file: + with rc_context({"svg.fonttype": "none"}): + fig.savefig(path, bbox_inches="tight") + else: + save_dendrogram_svg(fig, path) + finally: + plt.close(fig) + result = { + "classes": n, + "deep": args.deep, + "baseline": str(args.module_file or args.layout_file) if args.module_file or args.layout_file else None, + "path_vertices": sum(len(p.get_path().vertices) for ax in fig.axes for p in ax.patches), + "render_seconds": rendered - started, + "total_seconds": time.perf_counter() - started, + "svg_bytes": path.stat().st_size, + "svg_paths": path.read_text().count(" requested["trials"] or ( + source["status"] == "evaluated" and comparison["recorded_trials"] != requested["trials"] + ): + raise ValueError("CPU trial count disagrees with requested budget") + comparison["requested_settings"] = requested + return result, runtime, comparison + + +def archive(report, history, run_id, revision, profile, run_url=None, performance_valid=False, note=""): + if not IDENTIFIER.fullmatch(run_id) or not REVISION.fullmatch(revision): + raise ValueError("Require a safe run identifier and full hexadecimal source revision") + if not profile.strip(): + raise ValueError("Supply a descriptive profile") + if run_url is not None: + url = urlsplit(run_url) + if url.scheme != "https" or not url.netloc or url.username or url.password: + raise ValueError("Run URL must be HTTPS without credentials") + payload = Path(report).read_bytes() + source = json.loads(payload) + cpu = "pairs" in source and "required_candidate_ops" in source and "builds" not in source + if cpu: + evidence, hardware, comparison = cpu_evidence(source, Path(report).parent) + else: + evidence = project(source) + hardware = runtime_metadata(source, Path(report).parent) + record = { + "schema_version": 1, + "kind": "onnx_cpu_deployment" if cpu else "tensorrt_deployment", + "run_id": run_id, + "revision": revision, + "profile": profile, + "run_url": run_url, + "note": note, + "source_report_sha256": hashlib.sha256(payload).hexdigest(), + "runner_sha256": source.get("runner_sha256"), + "performance_valid": bool(performance_valid and evidence["status"] == "evaluated"), + "evidence": evidence, + "runtime": hardware, + "comparison": { + "inputs_sha256": source.get("inputs", {}).get("sha256"), + "manifest_sha256": source.get("manifest", {}).get("sha256"), + "settings": { + key: value + for key, value in source.get("settings", {}).items() + if key in ("trials", "warmup", "repeats", "memory_runs", "threads", "device", "pinned_host_io") + }, + "build_settings": { + role: { + key: value + for key, value in build.get("settings", {}).items() + if key in ("profiles", "fp16_allowed", "tf32_allowed", "workspace_bytes", "builder_optimization") + } + for role, build in source.get("builds", {}).items() + }, + }, + } + if cpu: + record["comparison"] = comparison + records = Path(history) / "records" + records.mkdir(parents=True, exist_ok=True) + destination = records / f"{run_id}.json" + if destination.exists(): + old = json.loads(destination.read_text()) + record["recorded_at"] = old["recorded_at"] + if old != record: + raise ValueError("Run identity already exists with different evidence or metadata") + return destination + record["recorded_at"] = datetime.now(UTC).isoformat() + encoded = (json.dumps(record, indent=2, allow_nan=False) + "\n").encode() + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=records, suffix=".tmp", delete=False) as stream: + temporary = Path(stream.name) + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + # Publish without overwriting a concurrent writer's immutable record. + os.link(temporary, destination) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return destination + + +def escape(value): + return html.escape(str(value), quote=True) + + +def display(value, scale=1): + return "undefined" if value is None else f"{number(value) * scale:.4f}" + + +def render(history): + history = Path(history) + records = [] + for path in sorted((history / "records").glob("*.json")): + record = json.loads(path.read_text()) + if record.get("schema_version") != 1 or record.get("kind") not in KINDS: + raise ValueError(f"Unsupported history record: {path.name}") + if not IDENTIFIER.fullmatch(record["run_id"]) or path.name != record["run_id"] + ".json": + raise ValueError("History filename and run identity differ") + records.append(record) + lines = [ + '', + '', + "Quantization evaluation history", + "", + "

Quantization evaluation history

", + "

Completed evaluation is not production acceptance. Compare quality and resource use on the intended hardware. " + "Device memory readings are device-wide snapshots, not per-process or transient peaks.

", + ] + for record in sorted(records, key=lambda r: (r["recorded_at"], r["run_id"]), reverse=True): + evidence = record["evidence"] + lines.append(f'
') + lines.append(f"

{escape(record['profile'])} — {escape(record['run_id'])}

") + lines.append( + f"

Status: {escape(evidence['status'])}; phase: {escape(evidence['phase'])}. " + f"Recorded: {escape(record['recorded_at'])}.

" + ) + lines.append( + f"

Revision {escape(record['revision'])}. " + f'Download compact record

' + ) + if record.get("run_url"): + url = urlsplit(record["run_url"]) + if url.scheme != "https" or not url.netloc or url.username or url.password: + raise ValueError("Unsafe history run URL") + lines.append(f'

Original workflow run

') + lines.append(f'

{escape(record["note"])}

') + hardware = record.get("runtime") + cpu = record["kind"] == "onnx_cpu_deployment" + device = hardware["environment"]["machine" if cpu else "gpu"] if hardware else "unrecorded" + lines.append( + f"

Measured {'CPU architecture' if cpu else 'GPU'}: {escape(device)}. " + f"Source report {escape(record['source_report_sha256'])}.

" + ) + if cpu: + requested = record["comparison"].get("requested_settings") + count = record["comparison"]["recorded_trials"] + budget = f"of {requested['trials']} requested" if requested else "(requested count was not retained)" + lines.append(f"

Recorded CPU trials: {escape(count)} {escape(budget)}.

") + for level in evidence["quality"]: + lines.append( + f"

{escape(level['name'])}: {escape(level['samples'])} samples, " + f"{escape(level['prediction_changes'])} changed predictions

" + ) + lines.append("") + for metric in METRICS: + values = level["metrics"][metric] + lines.append( + f"" + f"" + ) + lines.append("
MetricBaselineCandidateDelta ×100
{escape(METRIC_NAMES[metric])}{display(values['baseline'])}{display(values['candidate'])}{display(values['delta'], 100)}
") + if record["performance_valid"] and evidence["status"] == "evaluated" and cpu: + lines.append( + "

Resource measurements marked usable by the publisher; this is not independently certified. " + "CPU latency compares separate-process medians and excludes image decoding/preprocessing. " + "RSS includes process overhead; peak RSS is approximate.

" + "" + "" + ) + for trial in evidence["resources"]: + values = [str(trial["trial"]), display(trial["latency_ratio"])] + values += [ + display(trial["memory"][role][key], 1 / 2**20) + for key in ("resident_bytes", "peak_resident_bytes") + for role in ("baseline", "candidate") + ] + lines.append("" + "".join(f"" for value in values) + "") + lines.append("
TrialCPU median latency ratioBaseline RSS MiBCandidate RSS MiBBaseline approximate peak MiBCandidate approximate peak MiB
{escape(value)}

Ratios are candidate / baseline; below one is lower.

") + elif record["performance_valid"] and evidence["status"] == "evaluated": + lines.append("

Resource measurements marked usable by the publisher; this is not independently certified.

") + lines.append( + "" + "" + ) + for trial in evidence["resources"]: + a, b = [trial["memory"][role]["warm"] for role in ("baseline", "candidate")] + values = [ + trial["trial"], + display(trial["latency_ratio"]), + display(a["device_used_bytes"], 1 / 2**20), + display(b["device_used_bytes"], 1 / 2**20), + display(a["host_resident_bytes"], 1 / 2**20), + display(b["host_resident_bytes"], 1 / 2**20), + ] + lines.append("" + "".join(f"" for value in values) + "") + lines.append( + "
TrialPaired latency ratioBaseline device MiBCandidate device MiBBaseline host RSS MiBCandidate host RSS MiB
{escape(value)}

Latency ratios are candidate / baseline; below one is lower. " + "Inspect initialization snapshots in the compact record.

" + ) + else: + lines.append("

Resource readings excluded from performance comparisons.

") + if not evidence["quality"]: + lines.append("

Quality results unavailable.

") + lines.append("
") + if not records: + lines.append("

No archived runs.

") + lines.append("") + history.mkdir(parents=True, exist_ok=True) + (history / "index.html").write_text("\n".join(lines) + "\n") + return history / "index.html" + + +def main(): + parser = ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + add = sub.add_parser("archive") + for name in ("report", "history", "run-id", "revision", "profile"): + add.add_argument("--" + name, required=True) + add.add_argument("--run-url") + add.add_argument("--performance-valid", action="store_true") + add.add_argument("--note", default="") + sub.add_parser("render").add_argument("--history", required=True) + args = vars(parser.parse_args()) + command = args.pop("command") + archive(**args) if command == "archive" else render(**args) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/reporting/summarize.py b/dev/benchmarks/reporting/summarize.py new file mode 100644 index 00000000..c1f31bba --- /dev/null +++ b/dev/benchmarks/reporting/summarize.py @@ -0,0 +1,110 @@ +"""Render retained JSON benchmark reports as a repository/Actions summary.""" + +import json +import statistics +from argparse import ArgumentParser +from pathlib import Path + + +def summarize(directory: Path) -> str: + lines = [ + "# Dataset benchmark results", + "", + "| Run | Status | Device / precision | QT coverage | Accuracy by level | Parameter bytes | Peak CUDA MiB | " + "Median train epoch 3+ | Training wall time | Backbone parameters |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + reports = sorted(directory.rglob("report.json")) + quality_reports = [] + training_reports = 0 + for path in reports: + report = json.loads(path.read_text()) + if "models" in report or report.get("benchmark_kind") == "paired_quality": + quality_reports.append((path, report)) + continue + training_reports += 1 + accuracy = ", ".join(f"{value:.2%}" for value in report.get("level_accuracies", [])) or "—" + seconds = report.get("training_wall_seconds") + duration = f"{seconds:.2f}s" if seconds is not None else "—" + later_epochs = [ + phase["seconds"] for phase in report.get("phase_measurements", []) if phase["phase"] == "train" and phase["epoch"] >= 2 + ] + later_duration = ( + f"{statistics.median(later_epochs):.3f}s" + if later_epochs and report.get("phase_measurement_scope") + else "unverified" + if later_epochs + else "—" + ) + name = path.parent.relative_to(directory).as_posix() + device = f"{report.get('device', '?')} / {report.get('dtype', '?')}" + recipe = report.get("quantization_recipe") + quantization = ( + f"INT8 ({len(recipe['quantized_modules'])} Linear)" if recipe else "requested" if report.get("quantized_training") else "off" + ) + parameter_bytes = report.get("parameter_bytes", "—") + peak = report.get("peak_cuda_allocated_bytes") + peak_memory = ( + f"{peak / 2**20:.2f}" + if peak is not None and report.get("peak_cuda_memory_scope") + else "unverified" + if peak is not None + else "—" + ) + lines.append( + f"| {name} | {report['status']} | {device} | {quantization} | {accuracy} | {parameter_bytes} | " + f"{peak_memory} | {later_duration} | {duration} | {'frozen' if report.get('fine_tune') else 'trainable'} |" + ) + if not training_reports: + lines.append("| No reports produced | incomplete | — | — | — | — | — | — | — | — |") + if quality_reports: + lines.extend( + [ + "", + "## Paired held-out quality", + "", + "Differences are candidate minus baseline, multiplied by 100; they are not acceptance gates.", + "", + "| Pair / level | Status | Macro-F1 | Macro-Recall | Macro-Precision | Coverage | Theil's U |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for path, report in quality_reports: + name = path.parent.relative_to(directory).as_posix() + if report["status"] != "evaluated": + lines.append(f"| {name} | {report['status']} | — | — | — | — | — |") + continue + for level in report["levels"]: + values = [level["candidate_minus_baseline"].get(metric) for metric in ("f1", "recall", "precision", "coverage", "theilU")] + formatted = " | ".join("undefined" if value is None else f"{100 * value:+.3f}" for value in values) + label = str(level["name"]).replace("|", "\\|").replace("\n", " ").replace("\r", " ") + lines.append(f"| {name} / {label} | evaluated | {formatted} |") + lines.extend( + [ + "", + "Synthetic profiles require 100% oracle accuracy. Real-data runs marked `completed`", + "have no quality acceptance threshold yet; completion does not establish an improvement.", + "", + "Wall times include setup, training, validation, logging and checkpoints. Compare timings", + "only with matching hardware, dataset/configuration and timing scope. See JSON reports", + "for provenance, errors and explicit coverage flags. CPU results do not validate GPU behavior.", + "QT coverage counts quantized Linear modules; other operations may remain floating point.", + "Frozen backbone parameters do not imply evaluation mode: fine-tuning retains normal BatchNorm/dropout behavior.", + "Parameter bytes describe stored parameters. CUDA peaks cover training, excluding final held-out inference.", + "Older CUDA readings without a scope marker are unverified because logger resets could hide earlier peaks.", + "Later-epoch medians use timed training phases from epoch 3 onward, including loading, preprocessing and batch logging.", + "They exclude validation/figures/checkpoints, but may still include later compilation; they do not replace total wall time.", + ] + ) + return "\n".join(lines) + "\n" + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + args = parser.parse_args() + print(summarize(args.directory), end="") + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/reporting/svg_browser.py b/dev/benchmarks/reporting/svg_browser.py new file mode 100644 index 00000000..47fb6180 --- /dev/null +++ b/dev/benchmarks/reporting/svg_browser.py @@ -0,0 +1,71 @@ +"""Measure SVG/PNG decoding and forced rasterization in a disposable Chromium page.""" + +import argparse +import base64 +import json +import statistics +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("before", type=Path) + parser.add_argument("after", type=Path) + parser.add_argument("--output", type=Path, required=True, help="New artifact directory") + parser.add_argument("--repeats", type=int, default=5) + args = parser.parse_args() + if args.repeats < 1: + parser.error("--repeats must be positive") + try: + from playwright.sync_api import sync_playwright + except ImportError as error: + raise SystemExit("Install playwright and its Chromium browser in a disposable environment; see dendrogram.md") from error + + sources = {} + for name, path in (("before", args.before), ("after", args.after)): + if path.suffix.lower() not in (".svg", ".png"): + parser.error("Input files must be SVG or PNG") + mime = "image/svg+xml" if path.suffix.lower() == ".svg" else "image/png" + sources[name] = f"data:{mime};base64," + base64.b64encode(path.read_bytes()).decode() + args.output.mkdir(parents=True, exist_ok=False) + samples = {} + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + version = browser.version + for size in (1200, 4800): + for repeat in range(args.repeats): + for name in ("before", "after") if repeat % 2 == 0 else ("after", "before"): + context = browser.new_context() + try: + page = context.new_page() + result = page.evaluate( + """async ({svg, size}) => { + const image = new Image(); const start = performance.now(); + image.src = svg; + await image.decode(); const decoded = performance.now(); + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = size; + const ctx = canvas.getContext('2d'); + ctx.drawImage(image, 0, 0, size, size); + ctx.getImageData(0, 0, size, size); + const rastered = performance.now(); + return {decode_ms: decoded-start, raster_ms: rastered-decoded, + total_ms: rastered-start, png: canvas.toDataURL()}; + }""", + {"svg": sources[name], "size": size}, + ) + png = result.pop("png") + if repeat == 0: + (args.output / f"{name}-{size}.png").write_bytes(base64.b64decode(png.split(",", 1)[1])) + samples.setdefault(f"{name}-{size}", []).append(result) + finally: + context.close() + browser.close() + medians = {key: {metric: statistics.median(v[metric] for v in values) for metric in values[0]} for key, values in samples.items()} + report = {"chromium": version, "medians": medians, "samples": samples} + (args.output / "measurement.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(medians, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/summarize.py b/dev/benchmarks/summarize.py deleted file mode 100644 index ecb3a7aa..00000000 --- a/dev/benchmarks/summarize.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Render retained JSON benchmark reports as a repository/Actions summary.""" - -import json -from argparse import ArgumentParser -from pathlib import Path - - -def summarize(directory: Path) -> str: - lines = [ - "# Dataset benchmark results", - "", - "| Run | Status | Device / precision | Accuracy by level | Training wall time |", - "| --- | --- | --- | --- | --- |", - ] - reports = sorted(directory.rglob("report.json")) - for path in reports: - report = json.loads(path.read_text()) - accuracy = ", ".join(f"{value:.2%}" for value in report.get("level_accuracies", [])) or "—" - seconds = report.get("training_wall_seconds") - duration = f"{seconds:.2f}s" if seconds is not None else "—" - name = path.parent.relative_to(directory).as_posix() - device = f"{report.get('device', '?')} / {report.get('dtype', '?')}" - lines.append(f"| {name} | {report['status']} | {device} | {accuracy} | {duration} |") - if not reports: - lines.append("| No reports produced | incomplete | — | — | — |") - lines.extend( - [ - "", - "Synthetic profiles require 100% oracle accuracy. Real-data runs marked `completed`", - "have no quality acceptance threshold yet; completion does not establish an improvement.", - "", - "Wall times include setup, training, validation, logging and checkpoints. Compare timings", - "only with matching hardware, dataset/configuration and timing scope. See JSON reports", - "for provenance, errors and explicit coverage flags. CPU results do not validate GPU behavior.", - ] - ) - return "\n".join(lines) + "\n" - - -def main(): - parser = ArgumentParser(description=__doc__) - parser.add_argument("directory", type=Path) - args = parser.parse_args() - print(summarize(args.directory), end="") - - -if __name__ == "__main__": - main() diff --git a/dev/benchmarks/training.md b/dev/benchmarks/training.md new file mode 100644 index 00000000..8dbed365 --- /dev/null +++ b/dev/benchmarks/training.md @@ -0,0 +1,103 @@ +# Training and loading benchmarks + +Use prepared environments and fresh result directories. Start with the shared +CPU oracle in [the index](README.md), then use the representative profile below. +[Findings](../../docs/benchmarks.md) and [remaining work](../../docs/quantization-roadmap.md) +are maintained separately from these commands. + +## Representative paired EfficientNetV2 training profile + +```bash +CUDA_VISIBLE_DEVICES=0 PYTHONHASHSEED=0 \ +BENCHMARK_DATA_ROOT=/path/to/examples \ +BLAIR_CLASS_SPEC=/path/to/reviewed/class_spec.json \ +BENCHMARK_METRICS_PYTHON=/path/to/metrics-env/bin/python \ +bash dev/check-benchmarks.sh qt-efficientnet fresh-results +``` + +This runs pretrained EfficientNetV2-S with symmetric normalized flat/hierarchical +heads, full/frozen backbones, seeds 42/43/44 and floating BF16/native INT8: +24 sequential training processes followed by twelve held-out comparisons. +Defaults: five epochs, batch 32, 128px images, MuonAuxAdamW, learning rate 0.01, +CPU caching, zero workers and no compilation. Odd seeds reverse pair ordering. +Pretrained initialization can download weights; prepare its cache for offline use. + +| Override | Default | Selection | +| --- | --- | --- | +| `BENCHMARK_HEAD` | `both` | `flat`, `hierarchical`, `both` | +| `BENCHMARK_TRAINING_MODE` | `both` | `full`, `frozen`, `both` | +| `BENCHMARK_SEEDS` | `42 43 44` | Unique nonnegative integers | +| `BENCHMARK_EPOCHS` | `5` | Positive fixed schedule budget | +| `BENCHMARK_PYTHON` | `.venv/bin/python` | Prepared training interpreter | + +Choose budgets before inspecting held-out results. The profile evaluates `last.pt`, +retaining reports, checkpoints, epoch CSVs, predictions, manifests and failures. +`summary.md` reports Macro-F1, Macro-Recall, Macro-Precision, Coverage and Theil's U +at every level; deltas are multiplied by 100, including Theil's U. Undefined values +remain explicit. Failed pairs return nonzero without hiding the other pairs. + +## Individual configurations + +```bash +CUDA_VISIBLE_DEVICES=0 OMP_NUM_THREADS=1 .venv/bin/python -m dev.benchmarks.training.run \ + --dataset blair --data-root examples/blair \ + --class-spec examples/blair/blair_model/class_spec.json \ + --backbone efficientnet_v2_s --head hierarchical --hidden symmetric --normalized \ + --pretrained --image-size 128 --epochs 5 --batch-size 32 --seed 42 \ + --device cuda:0 --dtype bfloat16 --cache CPU --cache-workers 0 --num-workers 0 \ + --quantized-training --output fresh-int8-run +``` + +Omit `--quantized-training` for its floating pair. Match every other setting and +repeat seeds with alternating order. `--fine-tune` freezes backbone parameters +but retains normal training-mode BatchNorm/dropout; it is different from the +large-head capacity probe's evaluated frozen backbone. Compilation must be selected +identically for both precisions. `--help` lists the supported compilation modes. + +CPU float32 without QT verifies pipeline behavior; native INT8 execution needs +CUDA. Small random backbones verify execution, not useful convergence. Flat and +hierarchical comparisons retain the same leaf indices, reviewed taxonomy and splits. + +## Capacity and bottleneck probes + +Run each probe with `python -m dev.benchmarks.. --help` for its +arguments. These are focused diagnostics, not substitutes for paired training. + +| Module | Measures | +| --- | --- | +| `training.large_head_training` | EfficientNetV2 large-head setup, steps and memory; `--frozen` evaluates the backbone | +| `training.quantized_training` | Integer linear forward/backward and updates against floating compute | +| `training.quantization` | Separate CPU PTQ/QAT behavior | +| `data.loader` | Cached loader throughput | +| `data.cache` | Cache construction and host/shared-memory costs | +| `data.reader` | Streaming image decode/resize | +| `data.transfer` | Host/device transfer and overlap | + +```bash +OMP_NUM_THREADS=1 .venv/bin/python -m dev.benchmarks.data.loader +CUDA_VISIBLE_DEVICES=0 OMP_NUM_THREADS=1 \ + .venv/bin/python -m dev.benchmarks.training.quantized_training --width 2048 --batch-size 512 +``` + +## Training predictions to paired quality evaluation + +`training.training_predictions` converts existing runner predictions and dataset +manifests into the canonical paired input contract; `inference.quality_compare` +executes the five metrics in an explicitly selected `mini_metrics` environment. +The shared `qt-efficientnet` profile composes both. Use their `--help` for manual +re-evaluation. Match sample identities, class order, split and score semantics; +never substitute a classification-accuracy delta for the full metric comparison. + +## Measurement contract + +Use an otherwise idle allocation. Record hardware, versions, model/input hashes, +threads, workers, precision coverage and seeds. Separate cold setup/first step, +warm training, loading/transfers, allocated/reserved peaks and whole-call time. +Graph replay can trade setup or reserved memory for warm speed. Native QT retains +floating convolutions, gradients and optimizer states; report actual coverage. + +Epoch statistics from old runs with all statistic loggers disabled are invalid +convergence evidence, although independently evaluated final weights remain usable. +Historical optimizer/RNG state retention is limited; see [artifacts](../../docs/quantization-artifacts.md). +A few quality points are tolerable only with a substantial measured cost/speed or +memory benefit. Target hardware qualification remains open. diff --git a/dev/benchmarks/training/__init__.py b/dev/benchmarks/training/__init__.py new file mode 100644 index 00000000..145df043 --- /dev/null +++ b/dev/benchmarks/training/__init__.py @@ -0,0 +1 @@ +"""Development benchmarks: training.""" diff --git a/dev/benchmarks/training/large_head_training.py b/dev/benchmarks/training/large_head_training.py new file mode 100644 index 00000000..7fc0286a --- /dev/null +++ b/dev/benchmarks/training/large_head_training.py @@ -0,0 +1,216 @@ +"""Synthetic full-model or frozen-backbone training capacity probe; not convergence.""" + +import hashlib +import json +import platform +import statistics +import time +from argparse import ArgumentParser +from pathlib import Path + +import torch + +from mini_trainer.builders import BaseBuilder +from mini_trainer.hierarchical.model import HierarchicalClassifier +from mini_trainer.modeling import Classifier +from mini_trainer.modeling.classifier import classification_module +from mini_trainer.modeling.quantized_training import prepare_quantized_training +from mini_trainer.trainer import _optimizer_step +from mini_trainer.training import MuonAuxAdamW +from mini_trainer.training.compilation import compile_optimizer as prepare_compiled_optimizer +from mini_trainer.training.compilation import validate_optimizer_compilation + + +def run( + output, + classes=10000, + hierarchical=False, + frozen=False, + quantized=False, + batch_size=32, + image_size=128, + seed=42, + warmup=3, + steps=5, + device="cuda:0", + dtype="float16", + backbone="efficientnet_v2_s", + compile_optimizer=False, + optimizer_cudagraphs=False, +): + device = torch.device(device) + validate_optimizer_compilation(compile_optimizer, optimizer_cudagraphs, device) + if min(classes, batch_size, image_size, warmup, steps) < 1 or classes < 2 or batch_size < 2: + raise ValueError("Require at least two classes/samples and positive image size, warmup and steps") + if device.type not in ("cpu", "cuda") or dtype not in ("float32", "float16", "bfloat16"): + raise ValueError("Unsupported device or dtype") + if device.type == "cpu" and (quantized or dtype != "float32"): + raise ValueError("CPU is a float32 diagnostic only; native INT8 training requires CUDA") + if device.type == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("An accessible CUDA GPU is required") + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + torch.set_num_threads(1) + torch.manual_seed(seed) + report = { + "schema_version": 1, + "status": "running", + "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "settings": { + "classes": classes, + "hierarchical": hierarchical, + "frozen_backbone": frozen, + "quantized": quantized, + "batch_size": batch_size, + "image_size": image_size, + "seed": seed, + "warmup": warmup, + "steps": steps, + "device": str(device), + "dtype": dtype, + "backbone": backbone, + "hidden": "symmetric", + "normalized": True, + "compile_optimizer": compile_optimizer, + "optimizer_cudagraphs": optimizer_cudagraphs, + }, + "environment": {"platform": platform.platform(), "torch": torch.__version__}, + "warmup": [], + "steps": [], + "scope": ( + "Repeated fixed synthetic uint8 images and labels, MuonAuxAdamW and AMP. " + "Optimizer compilation/graph flags describe requested configuration, not verified graph replay. " + "Full-model or frozen-backbone updates; no cached embeddings. " + "No convergence, loader, checkpoint, distributed or target-hardware performance claims." + ), + } + + def synchronize(): + if device.type == "cuda": + torch.cuda.synchronize(device) + + try: + if device.type == "cuda": + torch.cuda.set_device(device) + torch.cuda.reset_peak_memory_stats(device) + report["environment"]["gpu"] = torch.cuda.get_device_name(device) + # Independent CPU generator prevents quantization/setup RNG consumption + # from changing the synthetic workload between float and INT8 runs. + generator = torch.Generator().manual_seed(seed + 1) + images = torch.randint(0, 256, (batch_size, 3, image_size, image_size), generator=generator, dtype=torch.uint8) + labels = torch.randint(classes, (batch_size,), generator=generator) + report["input_sha256"] = hashlib.sha256(images.numpy().tobytes() + labels.numpy().tobytes()).hexdigest() + images, labels = images.to(device), labels.to(device) + cls = HierarchicalClassifier if hierarchical else Classifier + kwargs = {"sparse_masks": [torch.arange(classes, device=device) // 100]} if hierarchical else {} + model, preprocess = cls.build( + model_type=backbone, + num_classes=classes, + hidden=True, + normalized=True, + model_args={"pretrained": False}, + device=device, + resize_size=image_size, + **kwargs, + ) + head = classification_module(model) + report["head_trainable_parameters"] = sum(p.numel() for p in head.parameters() if p.requires_grad) + head_ids = {id(p) for p in head.parameters()} + if frozen: + for parameter in model.parameters(): + if id(parameter) not in head_ids: + parameter.requires_grad_(False) + # Do not retain the final floating parameter after INT8 preparation + # replaces it; a large classification head can dominate this probe. + del parameter + model.eval() + head.train() + else: + model.train() + report["recipe"] = prepare_quantized_training(model) if quantized else None + if sum(p.numel() for p in head.parameters() if p.requires_grad) != report["head_trainable_parameters"]: + raise RuntimeError("Freezing/preparation changed the head's trainable parameter contract") + optimizer = BaseBuilder.build_optimizer(model, MuonAuxAdamW, lr=0.01, weight_decay=0.0) + if compile_optimizer: + prepare_compiled_optimizer(optimizer, cudagraphs=optimizer_cudagraphs) + scaler = BaseBuilder.build_scaler(device.type, enabled=device.type == "cuda" and dtype == "float16") + report["parameters"] = { + "total": sum(p.numel() for p in model.parameters()), + "trainable": sum(p.numel() for p in model.parameters() if p.requires_grad), + } + report["parameter_bytes"] = sum( + p.int_data.numel() + p.scale.numel() * p.scale.element_size() + if getattr(p, "_is_quantized_training", False) + else p.numel() * p.element_size() + for p in model.parameters() + ) + torch.manual_seed(seed + 2) + + def step(): + with torch.autocast(device.type, dtype=getattr(torch, dtype), enabled=dtype != "float32"): + scores = model(preprocess(images)) + values = scores if isinstance(scores, (tuple, list)) else [scores] + loss = torch.nn.functional.cross_entropy(values[0], labels) + if hierarchical: + loss = loss + torch.nn.functional.cross_entropy(values[1], labels // 100) + optimizer.zero_grad(set_to_none=True) + scaler.scale(loss).backward() + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), 5) + updated = _optimizer_step(optimizer, scaler) + return loss.detach(), updated + + for _ in range(warmup): + loss, updated = step() + report["warmup"].append({"loss": float(loss), "updated": updated}) + synchronize() + if device.type == "cuda": + report["setup_peak_allocated_bytes"] = torch.cuda.max_memory_allocated(device) + torch.cuda.reset_peak_memory_stats(device) + for _ in range(steps): + synchronize() + started = time.perf_counter() + loss, updated = step() + synchronize() + elapsed = time.perf_counter() - started + report["steps"].append({"seconds": elapsed, "loss": float(loss), "updated": updated}) + if device.type == "cuda": + report["measured_peak_allocated_bytes"] = torch.cuda.max_memory_allocated(device) + report["measured_peak_reserved_bytes"] = torch.cuda.max_memory_reserved(device) + if any(not s["updated"] or not torch.isfinite(torch.tensor(s["loss"])) for s in report["steps"]): + raise RuntimeError("Measured updates were skipped or nonfinite; timings are not a successful training comparison") + # Some valid heads retain trainable parameters for inactive branches + # (for example BatchNorm when normalization selects unit embeddings). + # Record those explicitly, while requiring an active head gradient. + report["unused_trainable_parameters"] = [name for name, p in model.named_parameters() if p.requires_grad and p.grad is None] + report["parameters"]["with_gradient"] = sum(p.numel() for p in model.parameters() if p.grad is not None) + if not any(p.requires_grad and p.grad is not None for p in head.parameters()): + raise RuntimeError("The classification head did not receive gradients") + if frozen and any(p.grad is not None for p in model.parameters() if not p.requires_grad): + raise RuntimeError("A frozen parameter received a gradient") + report["median_seconds_per_update"] = statistics.median(s["seconds"] for s in report["steps"]) + report["optimizer_steps"] = optimizer._step_count + report["status"] = "measured" + except Exception as error: + report.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + for name, default in (("classes", 10000), ("batch-size", 32), ("image-size", 128), ("seed", 42), ("warmup", 3), ("steps", 5)): + parser.add_argument("--" + name, type=int, default=default) + for flag in ("hierarchical", "frozen", "quantized", "compile-optimizer", "optimizer-cudagraphs"): + parser.add_argument("--" + flag, action="store_true") + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--dtype", choices=["float32", "float16", "bfloat16"], default="float16") + parser.add_argument("--backbone", default="efficientnet_v2_s") + run(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/training/performance.py b/dev/benchmarks/training/performance.py new file mode 100644 index 00000000..503a01e8 --- /dev/null +++ b/dev/benchmarks/training/performance.py @@ -0,0 +1,73 @@ +"""Preserve benchmark timing and memory across the logger's phase resets.""" + +import json +import time +from pathlib import Path + +import torch + +from mini_trainer.logging import MultiLogger + + +class BenchmarkLogger(MultiLogger): + def __init__(self, *args, measurement_device="cpu", **kwargs): + self.measurement_device = torch.device(measurement_device) + self.peak_allocated_bytes = 0 + self.phase_measurements = [] + self.phase_active = False + self.phase_peak_allocated_bytes = 0 + super().__init__(*args, **kwargs) + + def _reset_cuda_memory_stats(self): + if self.measurement_device.type == "cuda": + peak = torch.cuda.max_memory_allocated(self.measurement_device) + self.peak_allocated_bytes = max(self.peak_allocated_bytes, peak) + if self.phase_active: + self.phase_peak_allocated_bytes = max(self.phase_peak_allocated_bytes, peak) + super()._reset_cuda_memory_stats() + + def _synchronize(self): + if self.measurement_device.type == "cuda": + torch.cuda.synchronize(self.measurement_device) + + def start_timing(self): + self._synchronize() + super().start_timing() + self.phase_started = time.perf_counter() + self.phase_active = True + self.phase_peak_allocated_bytes = 0 + + def stop_timing(self): + self._synchronize() + self.phase_measurements.append( + { + "epoch": self._epoch, + "phase": self._type, + "seconds": time.perf_counter() - self.phase_started, + "peak_cuda_allocated_bytes": ( + max(self.phase_peak_allocated_bytes, torch.cuda.max_memory_allocated(self.measurement_device)) + if self.measurement_device.type == "cuda" + else None + ), + } + ) + self.phase_active = False + super().stop_timing() + + def finish(self): + super().finish() # Captures the last peak before the final reset. + if self.output_dir is not None: + Path(self.output_dir, "performance.json").write_text( + json.dumps( + { + "phases": self.phase_measurements, + "peak_cuda_allocated_bytes": self.peak_allocated_bytes if self.measurement_device.type == "cuda" else None, + "phase_scope": ( + "synchronized batch loop including loader, preprocessing, compute and batch logging; " + "excludes figures and checkpoints" + ), + }, + indent=2, + ) + + "\n" + ) diff --git a/dev/benchmarks/training/quantization.py b/dev/benchmarks/training/quantization.py new file mode 100644 index 00000000..7db4ae93 --- /dev/null +++ b/dev/benchmarks/training/quantization.py @@ -0,0 +1,210 @@ +"""Compare a recorded float checkpoint with PTQ and short QAT on training-only data.""" + +import hashlib +import json +import random +import subprocess +import time +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader, TensorDataset + +from mini_trainer.data import get_inference_dataloader +from mini_trainer.hierarchical.loss import MultiLevelWeightedCrossEntropyLoss +from mini_trainer.logging import MultiLogger +from mini_trainer.modeling import Classifier, EMATeacher +from mini_trainer.modeling.quantization import load_int8, prepare_int8 +from mini_trainer.trainer import train_one_epoch + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def run(baseline, output, *, data_root=None, calibration_samples=256, qat_epochs=2, batch_size=32): + baseline, output = Path(baseline), Path(output) + if output.exists(): + raise FileExistsError(output) + if batch_size < 2 or qat_epochs < 1 or calibration_samples < batch_size: + raise ValueError("Require batch_size >= 2, qat_epochs >= 1 and calibration_samples >= batch_size.") + baseline_report = json.loads((baseline / "report.json").read_text()) + dataset = baseline_report["dataset"] + synthetic = dataset == "synthetic" + root = baseline / "data" if synthetic else Path(data_root) if data_root else None + if root is None: + raise ValueError("Real datasets require --data-root.") + manifest_path = baseline / "data/manifest.json" if synthetic else baseline / "dataset_manifest.json" + weights = baseline / "training/weights/last.pt" + for path, key in ((manifest_path, "dataset_manifest_sha256"), (weights, "checkpoint_sha256")): + if digest(path) != baseline_report[key]: + raise ValueError(f"Baseline provenance mismatch: {path}") + manifest = json.loads(manifest_path.read_text()) + seed = baseline_report["seed"] + torch.manual_seed(seed) + train = [record for record in manifest["records"] if record["split"] == "train"] + random.Random(seed).shuffle(train) + # Keep full training batches: no duplicated or silently dropped examples. + count = min(calibration_samples, len(train)) // batch_size * batch_size + if count < batch_size: + raise ValueError("Not enough training samples for a full calibration batch.") + train = train[:count] + test = [record for record in manifest["records"] if record["split"] == "test"] + if not test: + raise ValueError("A held-out test split is required.") + for record in train + test: + if digest(root / record["path"]) != record["sha256"]: + raise ValueError(f"Dataset content changed: {record['path']}") + model, preprocess = Classifier.build(weights=str(weights), device=torch.device("cpu"), dtype=torch.float32) + model.eval() + size = {"synthetic": 8, "mnist": 28, "blair": 64}[dataset] + + def read(records): + _, loader = get_inference_dataloader( + images=[str(root / record["path"]) for record in records], + resize_size=size, + batch_size=batch_size, + num_workers=0, + device=torch.device("cpu"), + dtype=torch.float32, + ) + with torch.no_grad(): + return torch.cat([preprocess(images) for images in loader]) + + images = read(train) + targets = torch.tensor([record["targets"] if dataset == "blair" else record["label"] for record in train]) + example = images[:batch_size] + output.mkdir(parents=True) + provenance = {"split": "train", "seed": seed, "manifest_sha256": digest(manifest_path), "records": train} + (output / "calibration.json").write_text(json.dumps(provenance, indent=2) + "\n") + models = {"float": model} + coverage = {} + for mode in ("ptq", "qat"): + prepared = prepare_int8(model, example, qat=mode == "qat") + if mode == "ptq": + with torch.no_grad(): + for batch in images.split(batch_size): + prepared(batch) + else: + loader = DataLoader(TensorDataset(images, targets), batch_size=batch_size, shuffle=False) + optimizer = torch.optim.AdamW(prepared.parameters(), lr=1e-4, weight_decay=0) + scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=100000) + scaler = torch.amp.GradScaler("cpu", enabled=False) + with torch.no_grad(): + original_output = model(example) + criterion = ( + MultiLevelWeightedCrossEntropyLoss([v.shape[1] for v in original_output], torch.device("cpu"), torch.float32) + if isinstance(original_output, list) + else torch.nn.CrossEntropyLoss() + ) + teacher = EMATeacher(enable=False, total_steps=qat_epochs * len(loader)) + logger = MultiLogger(loader, loader, epochs=qat_epochs, output=None, name="qat", logger_cls=[]) + for epoch in range(qat_epochs): + train_one_epoch(prepared, teacher, criterion, optimizer, scaler, scheduler, loader, epoch, logger) + torch.save( + { + "model": prepared.state_dict(), + "optimizer": optimizer.state_dict(), + "lr_scheduler": scheduler.state_dict(), + "scaler": scaler.state_dict(), + "epoch": qat_epochs - 1, + }, + output / "qat_checkpoint.pt", + ) + prepared.convert().save( + output / mode, + example, + preprocessing={ + "source_checkpoint_sha256": digest(weights), + "resize_size": size, + "recipe": "Classifier.build checkpoint preprocessing; RGB uint8 input", + }, + calibration=provenance, + ) + models[mode], coverage[mode] = load_int8(output / mode).lower(example) + # Test images enter the process only after conversion and all training finish. + heldout = read(test) + labels = np.array([record["targets"] if dataset == "blair" else [record["label"]] for record in test]).T + results = {} + for name, inference in models.items(): + collected = [] + with torch.no_grad(): + inference(example) # Warm up outside timing. + started = time.perf_counter() + for batch in heldout.split(batch_size): + n = len(batch) + if n < batch_size: + batch = torch.cat([batch, batch[:1].expand(batch_size - n, *batch.shape[1:])]) + scores = inference(batch) + collected.append([value[:n] for value in (scores if isinstance(scores, list) else [scores])]) + seconds = time.perf_counter() - started + scores = [torch.cat(level).numpy() for level in zip(*collected, strict=True)] + if not all(np.isfinite(level).all() for level in scores): + raise ValueError(f"Nonfinite {name} predictions") + accuracies = [float((level.argmax(1) == truth).mean()) for level, truth in zip(scores, labels, strict=True)] + np.savez( + output / f"{name}_predictions.npz", + **{f"scores_{i}": v for i, v in enumerate(scores)}, + labels=labels, + paths=np.array([record["path"] for record in test]), + ) + results[name] = { + "level_accuracies": accuracies, + "test_inference_seconds": seconds, + "artifact_bytes": weights.stat().st_size if name == "float" else (output / name / "model.pt2").stat().st_size, + } + repository = Path(__file__).resolve().parents[3] + source_digest = hashlib.sha256() + for source in sorted((repository / "mini_trainer").rglob("*.py")) + sorted((repository / "dev/benchmarks").rglob("*.py")): + source_digest.update(str(source.relative_to(repository)).encode()) + source_digest.update(source.read_bytes()) + report = { + "source_sha256": source_digest.hexdigest(), + "lock_sha256": digest(repository / "uv.lock"), + "schema_version": 1, + "dataset": dataset, + "seed": seed, + "device": "cpu", + "amp": False, + "batch_size": batch_size, + "threads": torch.get_num_threads(), + "calibration_samples": count, + "qat_epochs": qat_epochs, + "qat_optimizer": {"name": "AdamW", "lr": 1e-4, "weight_decay": 0}, + "baseline_report_sha256": digest(baseline / "report.json"), + "checkpoint_sha256": digest(weights), + "calibration_sha256": digest(output / "calibration.json"), + "results": results, + "lowering": coverage, + "git_revision": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, cwd=repository).strip(), + "status": "passed" + if synthetic and all(r["level_accuracies"][0] == 1 for r in results.values()) + else "failed" + if synthetic + else "completed", + "timing_scope": "single warmed full-test inference pass including padding and concatenation; not a speedup claim", + } + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--data-root", type=Path) + parser.add_argument("--calibration-samples", type=int, default=256) + parser.add_argument("--qat-epochs", type=int, default=2) + parser.add_argument("--batch-size", type=int, default=32) + args = parser.parse_args() + torch.set_num_threads(1) + report = run(**vars(args)) + print(json.dumps(report["results"], indent=2)) + if report["status"] == "failed": + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/training/quantized_training.py b/dev/benchmarks/training/quantized_training.py new file mode 100644 index 00000000..304b6f5c --- /dev/null +++ b/dev/benchmarks/training/quantized_training.py @@ -0,0 +1,149 @@ +"""CUDA QT kernel probe using the opt-in model backend. + +Uses INT8 stored weights (no floating-point master copy), INT8 saved linear +inputs, and scaled INT8 forward/dgrad/wgrad GEMMs. Gradients and SGD update math +remain floating point. TorchAO stochastic rounding writes updates back to INT8. +SGD and AdamW are available; do not infer support for other optimizers. +""" + +import gc +import json +import math +import time +from argparse import ArgumentParser + +import torch + +from mini_trainer.modeling.quantized_training import IntegerLinear as IntegerLinear + + +def dependencies(): + from mini_trainer.modeling._quantized_training import TrainingWeight, quantize_int8_rowwise, scaled_int8_mm + + return TrainingWeight, quantize_int8_rowwise, scaled_int8_mm + + +class Layer(torch.nn.Module): + def __init__(self, width, quantized, dtype): + super().__init__() + self.quantized = quantized + weights = torch.randn(width, width, device="cuda", dtype=dtype) / width**0.5 + self.weight = torch.nn.Parameter(dependencies()[0].from_float(weights) if quantized else weights) + + def forward(self, inputs): + return torch.nn.functional.linear(inputs, self.weight) + + +def run( + width=4096, + batch_size=2048, + layers=4, + steps=10, + dtype="float16", + compiled=True, + optimizer_name="sgd", + weight_decay=0.0, + momentum=0.0, + epsilon=1e-4, +): + if not torch.cuda.is_available(): + raise RuntimeError("The QT probe requires an accessible CUDA GPU.") + results = {} + dependencies() + for quantized in (False, True): + torch.manual_seed(42) + gc.collect() + torch.cuda.empty_cache() + model = torch.nn.Sequential(*[Layer(width, quantized, getattr(torch, dtype)) for _ in range(layers)]) + if compiled: + model = torch.compile(model, fullgraph=True) + inputs = torch.randn(batch_size, width, device="cuda", dtype=getattr(torch, dtype)) + optimizer_cls = {"sgd": torch.optim.SGD, "adamw": torch.optim.AdamW}[optimizer_name] + optimizer_options = {"momentum": momentum} if optimizer_name == "sgd" else {"eps": epsilon} + optimizer = optimizer_cls(model.parameters(), lr=1e-3, weight_decay=weight_decay, foreach=False, **optimizer_options) + if compiled: + optimizer.step = torch.compile(optimizer.step, fullgraph=False) + + def step(model=model, inputs=inputs, optimizer=optimizer): + optimizer.zero_grad(set_to_none=True) + loss = model(inputs).float().square().mean() + loss.backward() + optimizer.step() + return loss + + warmup_losses = [] + for _ in range(3): + warmup_losses.append(float(step().detach())) + # This random linear-stack MSE must train every weight. AOT can + # otherwise report fast steps while silently dropping gradients + # for the experimental tensor subclass. + if any(parameter.grad is None or not bool(torch.count_nonzero(parameter.grad)) for parameter in model.parameters()): + raise RuntimeError( + "Missing or zero weight gradients in the QT probe; compiled tensor-subclass training is not verified. Try --eager." + ) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + started = time.perf_counter() + for _ in range(steps): + loss = step() + torch.cuda.synchronize() + seconds = (time.perf_counter() - started) / steps + results["int8" if quantized else dtype] = { + "seconds_per_step": seconds, + "peak_allocated_bytes": torch.cuda.max_memory_allocated(), + "loss": float(loss.detach()), + "warmup_losses": warmup_losses, + "weight_bytes": sum( + parameter.int_data.numel() + parameter.scale.numel() * parameter.scale.element_size() + if quantized + else parameter.numel() * parameter.element_size() + for parameter in model.parameters() + ), + } + if not math.isfinite(results["int8" if quantized else dtype]["loss"]): + raise RuntimeError("Nonfinite training loss; check optimizer precision, epsilon and learning rate.") + del step, model, inputs, optimizer, loss + return { + "device": torch.cuda.get_device_name(), + "torch": str(torch.__version__), + "width": width, + "batch_size": batch_size, + "layers": layers, + "steps": steps, + "compiled": compiled, + "seed": 42, + "scope": "synthetic square linear stack and MSE; warmed forward/backward/optimizer; excludes compilation and loading", + "optimizer": { + "name": optimizer_name, + "lr": 1e-3, + "weight_decay": weight_decay, + **({"momentum": momentum} if optimizer_name == "sgd" else {"eps": epsilon}), + }, + "results": results, + } + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--width", type=int, default=4096) + parser.add_argument("--batch-size", type=int, default=2048) + parser.add_argument("--layers", type=int, default=4) + parser.add_argument("--steps", type=int, default=10) + parser.add_argument("--dtype", choices=["float32", "float16"], default="float16") + parser.add_argument("--eager", action="store_true") + parser.add_argument("--optimizer-name", choices=["sgd", "adamw"], default="sgd") + parser.add_argument("--weight-decay", type=float, default=0.0) + parser.add_argument("--momentum", type=float, default=0.0) + parser.add_argument("--epsilon", type=float, default=1e-4, help="AdamW epsilon; 1e-4 avoids FP16 underflow") + args = vars(parser.parse_args()) + args["compiled"] = not args.pop("eager") + if min(args[key] for key in ("width", "batch_size", "layers", "steps")) < 1: + parser.error("Dimensions, layers and steps must be positive") + if args["optimizer_name"] != "sgd" and args["momentum"]: + parser.error("--momentum applies only to SGD") + torch.set_num_threads(1) + print(json.dumps(run(**args), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/benchmarks/run.py b/dev/benchmarks/training/run.py similarity index 50% rename from dev/benchmarks/run.py rename to dev/benchmarks/training/run.py index f7975bc2..07d7194a 100644 --- a/dev/benchmarks/run.py +++ b/dev/benchmarks/training/run.py @@ -6,27 +6,37 @@ import platform import subprocess import time -from argparse import ArgumentParser +from argparse import ArgumentParser, BooleanOptionalAction from datetime import UTC, datetime from importlib.metadata import version from pathlib import Path +import matplotlib import numpy as np import torch +from dev.benchmarks.data.datasets import prepare_real +from dev.benchmarks.data.synthetic import generate +from dev.benchmarks.models import NoAugmentationBuilder from mini_trainer.data import get_inference_dataloader from mini_trainer.hierarchical.integration import HierarchicalBuilder from mini_trainer.hierarchical.model import HierarchicalClassifier -from mini_trainer.modeling import Classifier +from mini_trainer.modeling import Classifier, classification_module from mini_trainer.train import main as train from mini_trainer.training import MuonAuxAdamW +from mini_trainer.training.compilation import MODEL_COMPILE_MODES, model_compile_options, validate_optimizer_compilation -from .datasets import prepare_real -from .models import NoAugmentationBuilder -from .synthetic import generate +from .performance import BenchmarkLogger + + +class BenchmarkBuilder(NoAugmentationBuilder): + @staticmethod + def build_logger(**kwargs): + return BenchmarkLogger(**kwargs) class HierarchicalBenchmarkBuilder(HierarchicalBuilder): + build_logger = staticmethod(BenchmarkBuilder.build_logger) build_augmentation = staticmethod(NoAugmentationBuilder.build_augmentation) @@ -42,15 +52,59 @@ def run( dataset: str = "synthetic", data_root: str | Path | None = None, class_spec: str | Path | None = None, + cuda_prefetch: bool = False, + quantized_training: bool = False, + compile: bool = False, + compile_optimizer: bool = False, + hidden: bool | int = 0, + backbone: str | None = None, + head: str = "auto", + normalized: bool | None = None, + image_size: int | None = None, + pretrained: bool = False, + fine_tune: bool = False, + batch_size: int = 32, + cache_workers: int | None = None, + model_profile: str = "default", + optimizer: str = "muon", + learning_rate: float | None = None, + compile_mode: str | None = None, + optimizer_cudagraphs: bool = False, ): + model_compile_options(compile, compile_mode) + validate_optimizer_compilation(compile_optimizer, optimizer_cudagraphs, device) + if model_profile not in ("default", "dense") or optimizer not in ("muon", "adamw", "sgd"): + raise ValueError("Unknown model or optimizer profile.") + if backbone is not None and model_profile != "default": + raise ValueError("Choose an explicit backbone or a model profile, not both.") + if head not in ("auto", "flat", "hierarchical"): + raise ValueError("Unknown head type.") + hierarchical = head == "hierarchical" or (head == "auto" and dataset == "blair") + if hierarchical and dataset != "blair": + raise ValueError("Hierarchical benchmarks require the reviewed Blair taxonomy.") + normalized = hierarchical if normalized is None else normalized + if hierarchical and not normalized: + raise ValueError("HierarchicalClassifier requires normalization.") + if image_size is not None and image_size < 1: + raise ValueError("Image size must be positive.") + if pretrained and backbone is None: + raise ValueError("Pretrained initialization requires an explicit backbone.") + if learning_rate is None: + learning_rate = 0.1 if dataset == "synthetic" else 0.01 + if not np.isfinite(learning_rate) or learning_rate <= 0: + raise ValueError("Learning rate must be finite and positive.") + matplotlib.use("Agg", force=True) + cache = "CPU" if cache == "RAM" else cache target_device = torch.device(device) precision = getattr(torch, dtype) if target_device.type not in ("cpu", "cuda") or dtype not in ("float32", "float16", "bfloat16"): raise ValueError("Benchmark profiles support CPU/CUDA with float32, float16 or bfloat16.") if target_device.type == "cpu" and (dtype == "float16" or cache == "CUDA"): raise ValueError("Use CUDA for the float16 or CUDA-cache profiles.") - if num_workers < 0 or epochs < 1: - raise ValueError("Workers must be nonnegative and epochs positive.") + if num_workers < 0 or epochs < 1 or hidden < 0 or batch_size < 2: + raise ValueError("Workers/hidden must be nonnegative, epochs positive and batch size at least two.") + if quantized_training and target_device.type != "cuda": + raise ValueError("Quantized training benchmark profiles require CUDA.") if target_device.type == "cuda": if not torch.cuda.is_available(): raise RuntimeError("CUDA profile requested but no accessible CUDA device is available.") @@ -59,6 +113,14 @@ def run( raise RuntimeError("The selected CUDA device does not support bfloat16.") torch.cuda.synchronize(target_device) torch.cuda.reset_peak_memory_stats(target_device) + repository = Path(__file__).resolve().parents[3] + revision = ( + subprocess.run(["git", "rev-parse", "HEAD"], cwd=repository, capture_output=True, text=True, check=False).stdout.strip() or None + ) + code_digest = hashlib.sha256() + for source in sorted((repository / "mini_trainer").rglob("*.py")) + sorted((repository / "dev/benchmarks").rglob("*.py")): + code_digest.update(str(source.relative_to(repository)).encode()) + code_digest.update(source.read_bytes()) output = Path(output).absolute() if output.exists(): raise FileExistsError(output) @@ -77,10 +139,18 @@ def run( manifest, spec = prepare_real(root, output, name=dataset, seed=seed, class_spec=Path(class_spec) if class_spec else None) manifest_path = output / "dataset_manifest.json" spec_path = output / "class_spec.json" - spec_path.write_text(json.dumps(spec, indent=2) + "\n") + # Preserve the taxonomy in the manifest, but train the flat head using + # its exact leaf indices so paired runs share both splits and class order. + training_spec = spec + if dataset == "blair" and not hierarchical: + training_spec = {"num_classes": spec["num_classes"][0], "cls2idx": spec["cls2idx"]["0"]} + spec_path.write_text(json.dumps(training_spec, indent=2) + "\n") size = 28 if dataset == "mnist" else 64 model_type = "dev.benchmarks.models:TinyConv" - hierarchical = dataset == "blair" + if model_profile == "dense": + model_type = "dev.benchmarks.models:DenseImageMLP" + model_type = backbone or model_type + size = image_size or size records = manifest["records"] train_records = [record for record in records if record["split"] != "test"] data_index = output / "train_index.json" @@ -106,33 +176,67 @@ def run( epochs=epochs, size=size, seed=seed, - builder=HierarchicalBenchmarkBuilder if hierarchical else NoAugmentationBuilder, + builder=HierarchicalBenchmarkBuilder if hierarchical else BenchmarkBuilder, ema=False, + quantized_training=quantized_training, + compile=compile, + compile_mode=compile_mode, + compile_optimizer=compile_optimizer, + optimizer_cudagraphs=optimizer_cudagraphs, model_builder_kwargs={ "model_type": model_type, - "hidden": False, - "normalized": hierarchical, + "fine_tune": fine_tune, + "fine_tune_dtype": torch.float32, + "hidden": hidden if hidden else False, + "normalized": normalized, + **({"model_args": {"pretrained": pretrained}} if backbone else {}), "cls": HierarchicalClassifier if hierarchical else Classifier, }, - dataloader_builder_kwargs={"batch_size": 32, "num_workers": num_workers, "data_index": str(data_index), "cache": cache}, - optimizer_builder_kwargs={"optimizer_cls": MuonAuxAdamW, "lr": 0.1 if dataset == "synthetic" else 0.01, "weight_decay": 0.0}, + dataloader_builder_kwargs={ + "batch_size": batch_size, + "num_workers": num_workers, + "data_index": str(data_index), + "cache": cache, + "cache_workers": cache_workers, + "cuda_prefetch": cuda_prefetch, + }, + optimizer_builder_kwargs={ + "optimizer_cls": {"muon": MuonAuxAdamW, "adamw": torch.optim.AdamW, "sgd": torch.optim.SGD}[optimizer], + "lr": learning_rate, + "weight_decay": 0.0, + **({"momentum": 0.9} if optimizer == "sgd" else {}), + }, criterion_builder_kwargs={"label_smoothing": 0.0}, regularizer_builder_kwargs={"strength": 0.0}, lr_schedule_builder_kwargs={"warmup_epochs": 0.0}, - logger_builder_kwargs={"logger_cls": []}, + logger_builder_kwargs={"measurement_device": device}, ) if target_device.type == "cuda": torch.cuda.synchronize(target_device) elapsed = time.perf_counter() - started - peak_memory = torch.cuda.max_memory_allocated(target_device) if target_device.type == "cuda" else None + performance = json.loads((output / "training/logs/performance.json").read_text()) + peak_memory = ( + max(performance["peak_cuda_allocated_bytes"], torch.cuda.max_memory_allocated(target_device)) + if target_device.type == "cuda" + else None + ) weights = output / "training/weights/last.pt" - model, preprocess = Classifier.build(weights=str(weights), device=target_device, dtype=torch.float32) + model, preprocess = Classifier.build( + weights=str(weights), + device=target_device, + dtype=torch.float32, + **({"model_args": {"pretrained": False}} if backbone else {}), + ) model.eval() + quantization_recipe = getattr(model, "_quantized_training_recipe", None) + if quantized_training and not quantization_recipe: + raise RuntimeError("Quantized training recipe was not restored from the checkpoint.") test_records = [record for record in records if record["split"] == "test"] _, loader = get_inference_dataloader( images=[str(root / record["path"]) for record in test_records], resize_size=size, - batch_size=32, + batch_size=batch_size, + cuda_prefetch=cuda_prefetch, num_workers=0, device=target_device, dtype=torch.float32, @@ -154,14 +258,6 @@ def run( arrays.update({f"scores_{level}": scores for level, scores in enumerate(scores_by_level)}) arrays.update({f"labels_{level}": labels for level, labels in enumerate(labels_by_level)}) np.savez(output / "predictions.npz", **arrays) - repository = Path(__file__).resolve().parents[2] - revision = ( - subprocess.run(["git", "rev-parse", "HEAD"], cwd=repository, capture_output=True, text=True, check=False).stdout.strip() or None - ) - code_digest = hashlib.sha256() - for source in sorted((repository / "mini_trainer").rglob("*.py")) + sorted(Path(__file__).parent.glob("*.py")): - code_digest.update(str(source.relative_to(repository)).encode()) - code_digest.update(source.read_bytes()) result = { "schema_version": 1, "status": "passed" if dataset == "synthetic" and accuracy == 1.0 else "failed" if dataset == "synthetic" else "completed", @@ -179,11 +275,39 @@ def run( "cuda_cache": cache == "CUDA", "ema": False, "distributed": False, - "quantization": False, + "quantization": bool(quantization_recipe), "augmentation": False, "onnx": False, }, "dataset": dataset, + "model_profile": model_profile, + "backbone": model_type, + "head": "hierarchical" if hierarchical else "flat", + "normalized": normalized, + "image_size": size, + "pretrained": pretrained, + "fine_tune": fine_tune, + "backbone_floating_dtype": "float32", + "backbone_training_mode": "train", + "hidden_width": classification_module(model).preclassification_size, + "optimizer": optimizer, + "learning_rate": learning_rate, + "momentum": 0.9 if optimizer == "sgd" else None, + "quantization_recipe": quantization_recipe, + "compile": compile, + "compile_mode": compile_mode, + "compile_optimizer": compile_optimizer, + "optimizer_cudagraphs": optimizer_cudagraphs, + "hidden": hidden, + "batch_size": batch_size, + "cache_workers": cache_workers, + "cuda_prefetch": cuda_prefetch, + "parameter_bytes": sum( + parameter.int_data.numel() + parameter.scale.numel() * parameter.scale.element_size() + if getattr(parameter, "_is_quantized_training", False) + else parameter.numel() * parameter.element_size() + for parameter in model.parameters() + ), "level_accuracies": accuracies, "split_counts": {split: sum(record["split"] == split for record in records) for split in ("train", "val", "test")}, "seed": seed, @@ -192,7 +316,13 @@ def run( "chance_accuracy": 1 / scores_by_level[0].shape[1], "test_accuracy": accuracy, "training_wall_seconds": elapsed, - "training_wall_scope": "setup, training, validation, logging and checkpoints", + "phase_measurements": performance["phases"], + "phase_measurement_scope": performance["phase_scope"], + "phase_peak_memory_scope": "maximum across all batch resets within each timed phase", + "peak_cuda_memory_scope": ( + "maximum across logger batch/phase resets and post-training allocation; excludes final held-out inference" + ), + "training_wall_scope": "setup, training, validation, logging and checkpoints; includes first-use compilation/autotuning", "num_workers_requested": num_workers, "num_workers": 0 if cache == "CUDA" else num_workers, "cache": cache, @@ -203,15 +333,23 @@ def run( "amp_exercised": precision != torch.float32, "deterministic_algorithms": torch.are_deterministic_algorithms_enabled(), "torch_threads": torch.get_num_threads(), + "torchinductor_compile_threads_env": os.environ.get("TORCHINDUCTOR_COMPILE_THREADS"), "python": platform.python_version(), "platform": platform.platform(), "cuda_version": torch.version.cuda, "cudnn_version": torch.backends.cudnn.version() if target_device.type == "cuda" else None, - "versions": {name: version(name) for name in ("torch", "torchvision", "numpy", "mini_trainer")}, + "versions": { + name: version(name) + for name in ( + ("torch", "torchvision", "numpy", "mini_trainer", "torchao") + if quantization_recipe + else ("torch", "torchvision", "numpy", "mini_trainer") + ) + }, "dataset_manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), "checkpoint_sha256": hashlib.sha256(weights.read_bytes()).hexdigest(), "score_semantics": "model_eval_forward", - "class_mapping": model.fc.metadata["cls2idx"], + "class_mapping": classification_module(model).metadata["cls2idx"], "test_used_for_training_or_selection": False, } (output / "report.json").write_text(json.dumps(result, indent=2) + "\n") @@ -229,8 +367,33 @@ def main(): parser.add_argument("--threads", type=int, default=1) parser.add_argument("--device", default="cpu") parser.add_argument("--dtype", choices=["float32", "float16", "bfloat16"], default="float32") - parser.add_argument("--cache", choices=["NONE", "RAM", "CUDA"], default="NONE") + parser.add_argument("--cache", choices=["NONE", "CPU", "RAM", "CUDA"], default="NONE") parser.add_argument("--num-workers", type=int, default=0) + parser.add_argument("--cache-workers", type=int) + parser.add_argument("--quantized-training", action="store_true") + parser.add_argument("--cuda-prefetch", action="store_true") + parser.add_argument("--compile", action="store_true") + parser.add_argument("--compile-mode", choices=MODEL_COMPILE_MODES) + parser.add_argument("--compile-optimizer", action="store_true") + parser.add_argument("--optimizer-cudagraphs", action="store_true") + parser.add_argument("--model-profile", choices=["default", "dense"], default="default") + parser.add_argument("--optimizer", choices=["muon", "adamw", "sgd"], default="muon") + parser.add_argument("--learning-rate", type=float) + parser.add_argument( + "--hidden", + type=lambda value: True if value == "symmetric" else int(value), + default=0, + help="0 disables the hidden layer; symmetric uses the backbone width; otherwise a positive width.", + ) + parser.add_argument("--backbone", help="Registered model name, for example efficientnet_v2_s.") + parser.add_argument("--head", choices=["auto", "flat", "hierarchical"], default="auto") + parser.add_argument("--normalized", action=BooleanOptionalAction, default=None) + parser.add_argument("--image-size", type=int) + parser.add_argument("--pretrained", action="store_true", help="Allow downloading pretrained backbone weights.") + parser.add_argument( + "--fine-tune", action="store_true", help="Freeze backbone parameters; retain normal training modes and FP32 storage." + ) + parser.add_argument("--batch-size", type=int, default=32) parser.add_argument( "--allow-nondeterministic", action="store_true", @@ -256,6 +419,24 @@ def main(): dataset=args.dataset, data_root=args.data_root, class_spec=args.class_spec, + cuda_prefetch=args.cuda_prefetch, + quantized_training=args.quantized_training, + compile=args.compile, + compile_mode=args.compile_mode, + compile_optimizer=args.compile_optimizer, + optimizer_cudagraphs=args.optimizer_cudagraphs, + hidden=args.hidden, + backbone=args.backbone, + head=args.head, + normalized=args.normalized, + image_size=args.image_size, + pretrained=args.pretrained, + fine_tune=args.fine_tune, + batch_size=args.batch_size, + cache_workers=args.cache_workers, + model_profile=args.model_profile, + optimizer=args.optimizer, + learning_rate=args.learning_rate, ) except Exception as error: args.output.mkdir(parents=True, exist_ok=True) @@ -268,6 +449,24 @@ def main(): "cache": args.cache, "seed": args.seed, "epochs": args.epochs, + "cuda_prefetch": args.cuda_prefetch, + "quantized_training": args.quantized_training, + "compile": args.compile, + "compile_mode": args.compile_mode, + "compile_optimizer": args.compile_optimizer, + "optimizer_cudagraphs": args.optimizer_cudagraphs, + "hidden": args.hidden, + "backbone": args.backbone, + "head": args.head, + "normalized": args.normalized, + "image_size": args.image_size, + "pretrained": args.pretrained, + "fine_tune": args.fine_tune, + "batch_size": args.batch_size, + "cache_workers": args.cache_workers, + "model_profile": args.model_profile, + "optimizer": args.optimizer, + "learning_rate": args.learning_rate, "error": {"type": type(error).__name__, "message": str(error)}, } (args.output / "report.json").write_text(json.dumps(failure, indent=2) + "\n") diff --git a/dev/benchmarks/training/training_predictions.py b/dev/benchmarks/training/training_predictions.py new file mode 100644 index 00000000..6c46498d --- /dev/null +++ b/dev/benchmarks/training/training_predictions.py @@ -0,0 +1,167 @@ +"""Prepare paired training-run predictions for the five-metric quality evaluator.""" + +import csv +import json +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np + +from dev.benchmarks.inference.onnx_inference import file_hash +from dev.benchmarks.inference.quality_compare import COLUMNS, read_manifest, read_predictions, validate_dataset + + +def _is_sha256(value): + return isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value) + + +def _read_run(directory): + directory = Path(directory).resolve() + report_path = directory / "report.json" + report = json.loads(report_path.read_bytes()) + if ( + report.get("schema_version") != 1 + or report.get("status") not in ("passed", "completed", "failed") + or not all(report.get("coverage", {}).get(key) is True for key in ("training", "checkpoint_reload", "inference")) + or report.get("test_used_for_training_or_selection") is not False + or report.get("score_semantics") != "model_eval_forward" + or not _is_sha256(report.get("checkpoint_sha256")) + ): + raise ValueError("Require a completed training/reload/inference report with unused held-out test predictions") + mapping = report["class_mapping"] + if report.get("head") == "flat": + mappings = [mapping] + elif report.get("head") == "hierarchical" and set(mapping) == {str(i) for i in range(len(mapping))}: + mappings = [mapping[str(i)] for i in range(len(mapping))] + else: + raise ValueError("Require flat or contiguous hierarchical class mappings") + classes = [] + for level in mappings: + if not level or any(type(i) is not int for i in level.values()) or set(level.values()) != set(range(len(level))): + raise ValueError("Class indices must be unique contiguous integers starting at zero") + classes.append([name for name, _ in sorted(level.items(), key=lambda item: item[1])]) + inventory_path = directory / ("data/manifest.json" if report.get("dataset") == "synthetic" else "dataset_manifest.json") + if file_hash(inventory_path) != report.get("dataset_manifest_sha256"): + raise ValueError("Dataset manifest hash does not match the training report") + inventory = json.loads(inventory_path.read_bytes()) + records = [record for record in inventory["records"] if record["split"] == "test"] + paths = [record["path"] for record in records] + if not paths or len(set(paths)) != len(paths): + raise ValueError("Require nonempty, unique held-out image paths") + samples = [] + image_hashes = {} + for identifier, record in enumerate(sorted(records, key=lambda record: record["path"])): + targets = [record["label"]] if len(classes) == 1 else record.get("targets", []) + if len(targets) != len(classes) or any(type(i) is not int or not 0 <= i < len(c) for i, c in zip(targets, classes, strict=True)): + raise ValueError("Held-out targets do not match the class mappings") + digest = record.get("sha256", "") + if not _is_sha256(digest): + raise ValueError("Require held-out image SHA256 identifiers") + image_hashes[record["path"]] = digest + samples.append( + {"instance_id": identifier, "filename": record["path"], "labels": [c[i] for c, i in zip(classes, targets, strict=True)]} + ) + metadata = { + "schema_version": 1, + "split": "test", + "levels": [{"name": "leaf" if i == 0 else f"level_{i}", "classes": c} for i, c in enumerate(classes)], + "samples": samples, + "provenance": {"dataset_manifest_sha256": file_hash(inventory_path), "image_sha256": image_hashes}, + } + validate_dataset(metadata) + predictions_path = directory / "predictions.npz" + tables = [] + with np.load(predictions_path, allow_pickle=False) as data: + if not np.array_equal(data["paths"], paths): + raise ValueError("Prediction paths differ from the held-out manifest order") + expected_keys = {f"{key}_{level}" for level in range(len(classes)) for key in ("scores", "labels")} + if set(data.files) != expected_keys | {"scores", "labels", "paths"}: + raise ValueError("Prediction archive has missing or unexpected levels/arrays") + for key in ("scores", "labels"): + if not np.array_equal(data[key], data[f"{key}_0"]): + raise ValueError("Leaf prediction aliases disagree") + indices = {path: i for i, path in enumerate(paths)} + for level, names in enumerate(classes): + scores, labels = data[f"scores_{level}"], data[f"labels_{level}"] + targets = [r["label"] if len(classes) == 1 else r["targets"][level] for r in records] + if labels.dtype.kind not in "iu" or not np.array_equal(labels, targets): + raise ValueError("Prediction labels differ from the held-out manifest") + if scores.dtype.kind != "f" or scores.shape != (len(paths), len(names)): + raise ValueError("Require finite floating scores with one row per image and column per class") + # This is a fixed argmax comparison, not a confidence calibration + # claim. Softmax supplies a bounded confidence for threshold zero. + for sample in samples: + row = indices[sample["filename"]] + values = scores[row].astype(np.float64) + if not np.isfinite(values).all(): + raise ValueError("Require finite floating scores") + prediction = int(values.argmax()) + confidence = float(1 / np.exp(values - values[prediction]).sum()) + tables.append( + ( + sample["instance_id"], + sample["filename"], + level, + names[labels[row]], + names[prediction], + confidence, + 0.0, + ) + ) + provenance = { + "report_path": str(report_path), + "report_sha256": file_hash(report_path), + "predictions_sha256": file_hash(predictions_path), + "training_report": report, + "checkpoint_verification": "Checkpoint identifier is taken from the training report; no checkpoint is loaded.", + } + return metadata, tables, provenance + + +def prepare(baseline, candidate, output): + """Validate both inputs before creating a portable quality-evaluation bundle.""" + output = Path(output) + if output.exists(): + raise FileExistsError(output) + runs = {mode: _read_run(path) for mode, path in (("baseline", baseline), ("candidate", candidate))} + metadata = runs["baseline"][0] + other = runs["candidate"][0] + if metadata["levels"] != other["levels"] or metadata["samples"] != other["samples"]: + raise ValueError("Paired runs must have identical held-out identities, labels and ordered class mappings") + if metadata["provenance"]["image_sha256"] != other["provenance"]["image_sha256"]: + raise ValueError("Paired held-out image hashes differ") + metadata["provenance"].update( + adapter_sha256=file_hash(__file__), + policy="Fixed argmax, softmax confidence, threshold zero; no threshold tuning or abstention.", + scope="Saved test predictions only; does not rerun inference, verify source images or establish performance/acceptance.", + ) + output.mkdir(parents=True) + for mode, (_, rows, provenance) in runs.items(): + path = output / f"{mode}.csv" + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(COLUMNS) + writer.writerows(rows) + metadata[mode] = { + "path": path.name, + "sha256": file_hash(path), + "classes": [s["classes"] for s in metadata["levels"]], + "provenance": provenance, + } + read_predictions(path, metadata[mode], metadata) + manifest = output / "manifest.json" + manifest.write_text(json.dumps(metadata, indent=2, allow_nan=False) + "\n") + read_manifest(manifest) + return manifest + + +def main(): + parser = ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True, help="Completed dataset benchmark directory") + parser.add_argument("--candidate", type=Path, required=True, help="Paired dataset benchmark directory") + parser.add_argument("--output", type=Path, required=True, help="New directory for CSVs and quality manifest") + print(prepare(**vars(parser.parse_args()))) + + +if __name__ == "__main__": + main() diff --git a/dev/check-benchmarks.sh b/dev/check-benchmarks.sh index ef23e379..f2a12a9e 100644 --- a/dev/check-benchmarks.sh +++ b/dev/check-benchmarks.sh @@ -9,15 +9,59 @@ if [[ -e "$results" ]]; then echo 'Results directory must be new.' >&2 exit 2 fi -case "$mode" in cpu|gpu|real) ;; *) echo 'Mode must be cpu, gpu or real.' >&2; exit 2 ;; esac +case "$mode" in cpu|gpu|real|qt|qt-real|qt-dense|qt-large-batch|qt-cudagraphs|qt-optimizer-cudagraphs|qt-efficientnet) ;; *) echo 'Mode must be cpu, gpu, real, qt, qt-real, qt-dense, qt-large-batch, qt-cudagraphs, qt-optimizer-cudagraphs or qt-efficientnet.' >&2; exit 2 ;; esac +if [[ "$mode" == qt-efficientnet ]]; then + : "${BENCHMARK_DATA_ROOT:?Set BENCHMARK_DATA_ROOT to the directory containing blair/}" + : "${BLAIR_CLASS_SPEC:?Set BLAIR_CLASS_SPEC to a reviewed Blair class specification}" + : "${BENCHMARK_METRICS_PYTHON:?Set BENCHMARK_METRICS_PYTHON to an environment with mini_metrics installed}" + epochs="${BENCHMARK_EPOCHS:-5}" + [[ "$epochs" =~ ^[1-9][0-9]*$ ]] || { echo 'BENCHMARK_EPOCHS must be a positive integer.' >&2; exit 2; } + read -r -a seeds <<< "${BENCHMARK_SEEDS:-42 43 44}" + [[ "${#seeds[@]}" -gt 0 ]] || { echo 'BENCHMARK_SEEDS must not be empty.' >&2; exit 2; } + declare -A seen_seeds=() + for seed in "${seeds[@]}"; do + [[ "$seed" =~ ^(0|[1-9][0-9]*)$ ]] || { echo 'BENCHMARK_SEEDS must contain nonnegative integers.' >&2; exit 2; } + [[ -z "${seen_seeds[$seed]:-}" ]] || { echo 'BENCHMARK_SEEDS must be unique.' >&2; exit 2; } + seen_seeds[$seed]=1 + done + case "${BENCHMARK_HEAD:-both}" in + both) heads=(flat hierarchical) ;; + flat|hierarchical) heads=("$BENCHMARK_HEAD") ;; + *) echo 'BENCHMARK_HEAD must be both, flat or hierarchical.' >&2; exit 2 ;; + esac + case "${BENCHMARK_TRAINING_MODE:-both}" in + both) training_modes=(full frozen) ;; + full|frozen) training_modes=("$BENCHMARK_TRAINING_MODE") ;; + *) echo 'BENCHMARK_TRAINING_MODE must be both, full or frozen.' >&2; exit 2 ;; + esac + "$BENCHMARK_METRICS_PYTHON" -c 'import mini_metrics' # Fail before training if the optional environment is unavailable. +fi mkdir -p -- "$results" status=0 run_profile() { local profile="$1" shift - if ! OMP_NUM_THREADS=1 "$benchmark_python" -m dev.benchmarks.run --output "$results/$profile" "$@" > "$results/$profile.log" 2>&1; then + if OMP_NUM_THREADS=1 MPLBACKEND=Agg TORCHINDUCTOR_COMPILE_THREADS="${TORCHINDUCTOR_COMPILE_THREADS:-1}" \ + "$benchmark_python" -m dev.benchmarks.training.run --output "$results/$profile" "$@" > "$results/$profile.log" 2>&1; then + return + else + local exit_code="$?" echo "Benchmark failed: $profile; see $results/$profile.log" >&2 status=1 + if [[ ! -f "$results/$profile/report.json" ]]; then + "$benchmark_python" - "$results/$profile/report.json" "$exit_code" "$@" <<'PY_REPORT' +import json +import sys +from pathlib import Path +path = Path(sys.argv[1]) +path.parent.mkdir(parents=True, exist_ok=True) +path.write_text(json.dumps({ + "schema_version": 1, "status": "failed", "arguments": sys.argv[3:], + "quantized_training": "--quantized-training" in sys.argv[3:], + "error": {"type": "ProcessFailure", "exit_code": int(sys.argv[2]), "message": "Process exited without a report; see profile log."}, +}, indent=2) + "\n") +PY_REPORT + fi fi } if [[ "$mode" == cpu ]]; then @@ -26,6 +70,113 @@ elif [[ "$mode" == gpu ]]; then for precision in float32 float16 bfloat16; do run_profile "synthetic-cuda-$precision" --device cuda:0 --dtype "$precision" --cache CUDA done +elif [[ "$mode" == qt ]]; then + run_profile synthetic-float --device cuda:0 --dtype float16 --cache CPU --cache-workers 0 + run_profile synthetic-int8 --device cuda:0 --dtype float16 --cache CPU --cache-workers 0 --quantized-training +elif [[ "$mode" == qt-dense ]]; then + : "${BENCHMARK_DATA_ROOT:?Set BENCHMARK_DATA_ROOT to the directory containing mnist/}" + for precision in float int8; do + quantization=() + if [[ "$precision" == int8 ]]; then quantization=(--quantized-training); fi + run_profile "mnist-dense-$precision" --dataset mnist --data-root "$BENCHMARK_DATA_ROOT/mnist" \ + --model-profile dense --optimizer sgd --learning-rate 0.3 --epochs 15 --batch-size 128 --compile \ + --device cuda:0 --dtype float16 --cache CPU --cache-workers 0 --allow-nondeterministic "${quantization[@]}" + done +elif [[ "$mode" == qt-large-batch || "$mode" == qt-cudagraphs || "$mode" == qt-optimizer-cudagraphs ]]; then + compile_mode=() + profile=mnist-large-batch + if [[ "$mode" == qt-cudagraphs || "$mode" == qt-optimizer-cudagraphs ]]; then + compile_mode=(--compile-mode reduce-overhead) + profile=mnist-cudagraphs + fi + if [[ "$mode" == qt-optimizer-cudagraphs ]]; then + compile_mode+=(--optimizer-cudagraphs) + profile=mnist-optimizer-cudagraphs + fi + : "${BENCHMARK_DATA_ROOT:?Set BENCHMARK_DATA_ROOT to the directory containing mnist/}" + for seed in 42 43 44; do + precisions=(float int8) + if (( seed % 2 )); then precisions=(int8 float); fi + for precision in "${precisions[@]}"; do + quantization=() + if [[ "$precision" == int8 ]]; then quantization=(--quantized-training); fi + run_profile "$profile-$precision-seed$seed" --dataset mnist --data-root "$BENCHMARK_DATA_ROOT/mnist" \ + --seed "$seed" --model-profile dense --optimizer sgd --learning-rate 0.3 --epochs 60 --batch-size 512 \ + --compile "${compile_mode[@]}" --compile-optimizer --device cuda:0 --dtype float16 --cache CPU --cache-workers 0 \ + --allow-nondeterministic "${quantization[@]}" + done + done +elif [[ "$mode" == qt-efficientnet ]]; then + for seed in "${seeds[@]}"; do + precisions=(float int8) + modes=("${training_modes[@]}") + if [[ "$seed" =~ [13579]$ ]]; then + precisions=(int8 float) + if [[ "${#modes[@]}" == 2 ]]; then modes=(frozen full); fi + fi + for head in "${heads[@]}"; do + for training_mode in "${modes[@]}"; do + extra=() + if [[ "$training_mode" == frozen ]]; then extra=(--fine-tune); fi + for precision in "${precisions[@]}"; do + quantization=() + if [[ "$precision" == int8 ]]; then quantization=(--quantized-training); fi + run_profile "blair-$head-$training_mode-$precision-seed$seed" \ + --dataset blair --data-root "$BENCHMARK_DATA_ROOT/blair" --class-spec "$BLAIR_CLASS_SPEC" \ + --backbone efficientnet_v2_s --head "$head" --hidden symmetric --normalized --pretrained \ + --seed "$seed" --epochs "$epochs" --batch-size 32 --image-size 128 --optimizer muon --learning-rate 0.01 \ + --device cuda:0 --dtype bfloat16 --cache CPU --cache-workers 0 --num-workers 0 --threads 1 \ + "${extra[@]}" "${quantization[@]}" + done + done + done + done + # Evaluate only after all training timings, in the explicitly selected metrics environment. + for seed in "${seeds[@]}"; do + for head in "${heads[@]}"; do + for training_mode in "${training_modes[@]}"; do + pair="blair-$head-$training_mode-seed$seed" + if "$benchmark_python" -m dev.benchmarks.training.training_predictions \ + --baseline "$results/blair-$head-$training_mode-float-seed$seed" \ + --candidate "$results/blair-$head-$training_mode-int8-seed$seed" \ + --output "$results/$pair-inputs" > "$results/$pair-quality.log" 2>&1 && \ + PYTHONHASHSEED=0 OMP_NUM_THREADS=1 "$BENCHMARK_METRICS_PYTHON" -m dev.benchmarks.inference.quality_compare \ + --manifest "$results/$pair-inputs/manifest.json" --output "$results/$pair-quality" \ + >> "$results/$pair-quality.log" 2>&1; then + : + else + echo "Quality evaluation failed: $pair; see $results/$pair-quality.log" >&2 + status=1 + "$benchmark_python" - "$results/$pair-quality/report.json" <<'PY_QUALITY_FAILURE' +import json +import sys +from pathlib import Path +path = Path(sys.argv[1]) +if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"schema_version": 1, "benchmark_kind": "paired_quality", "status": "failed", + "error": "Prediction preparation or evaluation failed; see the pair quality log."}) + "\n") +PY_QUALITY_FAILURE + fi + done + done + done +elif [[ "$mode" == qt-real ]]; then + : "${BENCHMARK_DATA_ROOT:?Set BENCHMARK_DATA_ROOT to the directory containing mnist/ and blair/}" + : "${BLAIR_CLASS_SPEC:?Set BLAIR_CLASS_SPEC to a reviewed Blair class specification}" + for dataset in mnist blair; do + extra=() + if [[ "$dataset" == blair ]]; then + extra=(--class-spec "$BLAIR_CLASS_SPEC" --hidden 64) + fi + for precision in float int8; do + quantization=() + if [[ "$precision" == int8 ]]; then quantization=(--quantized-training); fi + run_profile "$dataset-$precision" --dataset "$dataset" --data-root "$BENCHMARK_DATA_ROOT/$dataset" \ + --epochs 5 --device cuda:0 --dtype float16 --cache CPU --cache-workers 0 --allow-nondeterministic \ + "${extra[@]}" "${quantization[@]}" + done + done else : "${BENCHMARK_DATA_ROOT:?Set BENCHMARK_DATA_ROOT to the directory containing mnist/ and blair/}" : "${BLAIR_CLASS_SPEC:?Set BLAIR_CLASS_SPEC to a reviewed Blair class specification}" @@ -33,6 +184,6 @@ else run_profile mnist-cuda --dataset mnist --data-root "$BENCHMARK_DATA_ROOT/mnist" --epochs 5 --device cuda:0 --dtype float16 --cache CUDA --allow-nondeterministic run_profile blair-cuda --dataset blair --data-root "$BENCHMARK_DATA_ROOT/blair" --class-spec "$BLAIR_CLASS_SPEC" --epochs 5 --device cuda:0 --dtype float16 --cache CUDA --allow-nondeterministic fi -"$benchmark_python" -m dev.benchmarks.summarize "$results" > "$results/summary.md" +"$benchmark_python" -m dev.benchmarks.reporting.summarize "$results" > "$results/summary.md" cat "$results/summary.md" exit "$status" diff --git a/dev/check-report-history.sh b/dev/check-report-history.sh new file mode 100644 index 00000000..6f219fe4 --- /dev/null +++ b/dev/check-report-history.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Prepare a compact artifact from a completed or failed target deployment. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +results="${1:?Supply the target results directory}" +history="${2:?Supply the compact history directory}" +: "${BENCHMARK_PYTHON:?Set BENCHMARK_PYTHON to a prepared Python 3.12+ executable}" +: "${BENCHMARK_RUN_ID:?Set a unique run and attempt identifier}" +: "${BENCHMARK_REVISION:?Set the full source commit hash}" +: "${BENCHMARK_PROFILE:?Describe the dataset, model and target configuration}" +report="$results/evaluation/report.json" +if [[ ! -f "$report" ]]; then report="$results/status.json"; fi +args=(--report "$report" --history "$history" --run-id "$BENCHMARK_RUN_ID" + --revision "$BENCHMARK_REVISION" --profile "$BENCHMARK_PROFILE" + --note "${BENCHMARK_NOTE:-}") +if [[ -n "${BENCHMARK_RUN_URL:-}" ]]; then args+=(--run-url "$BENCHMARK_RUN_URL"); fi +case "${BENCHMARK_PERFORMANCE_VALID:-false}" in + true) args+=(--performance-valid) ;; + false) ;; + *) echo 'BENCHMARK_PERFORMANCE_VALID must be true or false.' >&2; exit 2 ;; +esac +"$BENCHMARK_PYTHON" -m dev.benchmarks.reporting.report_history archive "${args[@]}" +"$BENCHMARK_PYTHON" -m dev.benchmarks.reporting.report_history render --history "$history" diff --git a/dev/check-tensorrt-deployment.sh b/dev/check-tensorrt-deployment.sh new file mode 100644 index 00000000..c2ffb99b --- /dev/null +++ b/dev/check-tensorrt-deployment.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Build engines on the target and evaluate them using explicitly prepared environments. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +results="${1:?Supply a new results directory}" +if [[ -e "$results" || -L "$results" ]]; then + echo 'Results directory must be new.' >&2 + exit 2 +fi +mkdir -p -- "$results" +phase=configuration +trap 'code=$?; printf "{\"phase\":\"%s\",\"exit_code\":%d}\n" "$phase" "$code" > "$results/status.json"' EXIT +: "${BENCHMARK_PYTHON:?Set BENCHMARK_PYTHON to an explicitly prepared TensorRT/CUDA Python}" +: "${BENCHMARK_METRICS_PYTHON:?Set BENCHMARK_METRICS_PYTHON to an environment with mini_metrics}" +: "${TRT_BASELINE_MODEL:?Set TRT_BASELINE_MODEL to the floating ONNX model}" +: "${TRT_CANDIDATE_MODEL:?Set TRT_CANDIDATE_MODEL to the calibrated candidate ONNX model}" +: "${TRT_INFERENCE_MANIFEST:?Set TRT_INFERENCE_MANIFEST to the held-out inference manifest}" +: "${TRT_INPUTS:?Set TRT_INPUTS to representative named preprocessed NPZ inputs}" +: "${TRT_PROFILES:?Set TRT_PROFILES to the shared TensorRT profile JSON}" +export OMP_NUM_THREADS="${BENCHMARK_THREADS:-1}" +export PYTHONHASHSEED=0 +phase=preflight +git rev-parse HEAD > "$results/revision.txt" +sha256sum -- dev/check-tensorrt-deployment.sh > "$results/harness.sha256" +"$BENCHMARK_PYTHON" -c 'import onnx, numpy, tensorrt, torch; assert torch.cuda.is_available(), "CUDA must be available"' > "$results/preflight.log" 2>&1 +"$BENCHMARK_METRICS_PYTHON" -c 'from mini_metrics.data import MetricDF; from mini_metrics.metrics import MacroF1, evaluate_file' >> "$results/preflight.log" 2>&1 +for role in baseline candidate; do + phase="build-$role" + model="$TRT_BASELINE_MODEL" + if [[ "$role" == candidate ]]; then model="$TRT_CANDIDATE_MODEL"; fi + "$BENCHMARK_PYTHON" -m dev.benchmarks.inference.tensorrt_build \ + --model "$model" --inputs "$TRT_INPUTS" --profiles "$TRT_PROFILES" \ + --output "$results/$role" --fp16 --device "${BENCHMARK_DEVICE:-0}" \ + --optimization "${TRT_OPTIMIZATION:-1}" --workspace-mib "${TRT_WORKSPACE_MIB:-1024}" \ + > "$results/build-$role.log" 2>&1 +done +phase=evaluation +"$BENCHMARK_PYTHON" -m dev.benchmarks.inference.tensorrt_deployment \ + --baseline-build "$results/baseline" --candidate-build "$results/candidate" \ + --manifest "$TRT_INFERENCE_MANIFEST" --inputs "$TRT_INPUTS" --output "$results/evaluation" \ + --metrics-python "$BENCHMARK_METRICS_PYTHON" --threads "$OMP_NUM_THREADS" \ + --device "${BENCHMARK_DEVICE:-0}" > "$results/evaluation.log" 2>&1 +phase=complete diff --git a/dev/ci_scope.py b/dev/ci_scope.py new file mode 100644 index 00000000..9bf8e713 --- /dev/null +++ b/dev/ci_scope.py @@ -0,0 +1,25 @@ +"""Select code checks for PRs; unreadable or empty diffs conservatively run them.""" + +import os +import subprocess + + +def agent_document(path): + return path in {"AGENTS.md", ".agents/.gitignore"} or (path.startswith(".agents/") and path.endswith(".md")) + + +def needs_checks(base, head): + if not base or not head: + return True + try: + changed = subprocess.check_output( + ["git", "diff", "--name-only", "--no-renames", "-z", f"{base}...{head}", "--"], stderr=subprocess.PIPE + ) + except subprocess.CalledProcessError: + return True + paths = [os.fsdecode(path) for path in changed.split(b"\0") if path] + return not paths or any(not agent_document(path) for path in paths) + + +if __name__ == "__main__": + print(f"run_checks={str(needs_checks(os.environ.get('BASE_SHA'), os.environ.get('HEAD_SHA'))).lower()}") diff --git a/dev/ucloud/README.md b/dev/ucloud/README.md new file mode 100644 index 00000000..3d9cd5f4 --- /dev/null +++ b/dev/ucloud/README.md @@ -0,0 +1,851 @@ +# UCloud global_lepi training comparison + +Run inside **one allocated node**, with 1, 2, 4 or 8 visible GPUs. More than one +GPU uses one `torchrun` process per GPU and the package's existing DDP trainer +(including SyncBatchNorm). One GPU uses ordinary training. Do not invoke the +launcher once per GPU or wrap it in a multi-task `srun`. + +Allocate the job manually in the UCloud web interface, connect to that node over +SSH, install the environments below, then run the launcher from the shell. Local +Slurm inside the allocation is unnecessary and cannot allocate further nodes. +Use `tmux` or another persistent shell if an SSH disconnect would stop the process; +this does not extend the allocation's lifetime. + +UCloud's [PyTorch app](https://docs.cloud.sdu.dk/Apps/pytorch.html) supports a Bash +batch script and selection of GPU machine types. Mounted volumes appear under +[`/work`](https://docs.cloud.sdu.dk/guide/submitting.html). Use the actual paths +shown in your job. No Gefion account, partition, GPU type or Slurm directives are +assumed here. The scripts perform no installation, submission or paid allocation. + +## Comparison and controls + +The source configuration is `publication/experiments/config_temp.yaml` (singular +`publication`), a local experiment file. This protocol keeps 10 epochs, FP16 AMP, +class weighting, 0.25 warmup epochs and eight loader workers per rank. It replaces +ViT with torchvision `efficientnet_v2_s` (EfficientNetV2-S) and fixes the +`HierarchicalClassifier` head, species/genus/family, equal loss weights, normalized +prototypes, hidden layer, dropout 0.1, MuonAuxAdamW, learning rate 0.001, weight decay +0.01, regularizer 0.1 and automatic smoothing. Resizing is explicitly 384 pixels, +the backbone's preferred crop size. The initial pretrained backbone and randomly +initialized head are saved **once per seed by master**, then loaded by both branches. +Smoothing values per hierarchy level are recorded in `dataset.json`. + +**Batch size 64 is global**, so it becomes 32/16/8 per rank for 2/4/8 GPUs. Learning +rate is not scaled. Changing GPU count still changes SyncBatchNorm, sampling and +kernel behavior; compare rows within an allocation, not as identical trajectories +across GPU counts. Three paired seeds are the default. Run order is shuffled within +each seed using a reproducible schedule. RNG streams are not guaranteed identical +across implementations, but starting model tensors are identical. No resume or +checkpoint averaging is used in measured runs. + +| Variant | Branch | Change from eager control | +| --- | --- | --- | +| `master_eager` | master | Baseline | +| `quant_eager` | quant | Branch changes with all optional acceleration off | +| `quant_prefetch` | quant | CUDA transfer lookahead | +| `quant_compile_model` | quant | Default model compilation | +| `quant_compile_optimizer` | quant | Optimizer compilation | +| `quant_combined` | quant | Prefetch plus both compilation options | + +Optional variant names: `quant_model_graphs` (reduce-overhead model mode), +`quant_optimizer_graphs` (optimizer graph replay), and **`quant_int8` (one GPU +only)**. For a single-GPU INT8 comparison, set `gpus: 1` and include at least +`master_eager`, `quant_eager`, `quant_int8`. The runner rejects INT8 with DDP. +These are target-machine qualification experiments; a supported flag is not a +guarantee that a compiled configuration succeeds or is faster on this hierarchy. +Failures remain visible. No automatic fallback is used. + +Muon, normalized heads, EMLA/class weighting, regularization and smoothing already +exist on the pinned master. They are held constant rather than mislabeled as new +quant features. Their quality/tuning ablations belong to the separate +[feature-validation protocol](../../docs/training-feature-validation.md). +EMA is disabled. CPU PTQ/QAT are inference experiments. Full dataset RAM caching +is excluded because it replicates the dataset per DDP rank and can exhaust node +memory; both branches use uncached image loading. W&B is disabled to avoid requiring +credentials; local logs and checkpoints are retained. + +## Fresh job setup + +For the first bounded test, select one MIG in UCloud and mount the global_lepi +dataset. Internet access is needed for the source checkout, Python dependencies +and the first pretrained-weight download. CPU and RAM come with the selected +GPU product; the host totals reported inside the container are not your budget. + +Clone the branch containing the current harness, then install uv if the image +does not already provide it. Follow the [official uv installation instructions](https://docs.astral.sh/uv/getting-started/installation/). + +```bash +git clone --branch quant https://github.com/asgersvenning/mini_trainer.git /work/mini_trainer +# Only if `uv --version` is unavailable: +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + +cd /work/mini_trainer +bash dev/ucloud/setup.sh /work/YOUR_DATASET/YOUR_METADATA.parquet +``` + +The setup script reads the pinned commits from `qualification.json`, exports the +quant commit's lockfile into `/work/requirements-mt.txt`, creates two Python 3.12 +environments and checks their dependencies. It uses explicit indexes and required +hashes, stops on the first error, and generates `/work/qualification.json` with the +supplied dataset path and a new smoke-test output path. It does not start training +or touch the dataset. No editable package is installed, and the checkout stays on +the harness branch. Do not reset it to the older pinned package commits. + +`MT_WORK_ROOT` overrides `/work`, `MT_CONFIG` chooses another generated config path, +and `MT_TORCH_BACKEND` selects `cu126`, `cu130` (default) or `cu132`. A failed setup can +be rerun: it synchronizes only its dedicated `venvs/mt-master` and `venvs/mt-quant` +environments, removing leftover packages there. An existing generated config is +never overwritten. The UV download cache defaults to `/work/.cache/uv`; installation +time and disk space depend on the node and network and are outside the test budget. + +Once setup succeeds: + +```bash +cd /work/mini_trainer/dev/ucloud +export TORCH_HOME=/work/.cache/torch +bash launch.sh /work/qualification.json --stage plan +bash launch.sh /work/qualification.json --stage prepare && \ + bash launch.sh /work/qualification.json --stage train +bash launch.sh /work/qualification.json --stage summary +``` + +Preparation and both eager runs share a 30-minute wall-clock budget. It does not +terminate the UCloud job; choose the allocation lifetime separately, allowing for +installation. See the qualification and resource discussion below before expanding +the matrix. The manual setup below remains available for full-dataset comparisons. + +## Fresh environment setup (manual) + +Use the same Python and exact dependency versions for both packages. The runner +checks installed VCS commit IDs and dependency inventories. No editable installs: +running from the harness directory must import the selected installed branch. +The node image must provide a working NVIDIA driver and a C++ compiler (`c++`, or +the executable named by `CXX`); the preflight checks both CUDA execution and compiler +availability. Python wheels do not provide the NVIDIA kernel driver. +The default commits are the local branch tips inspected when this harness was +prepared, not a claim about the latest remote branches: + +```bash +MASTER_SHA=2ebbe39f2365bfea482d7ebc3d000cf4d2044918 +QUANT_SHA=4ea6b9613fc29a8e33ef72f28885f4e2a0e5906b +REPO_URL=https://github.com/asgersvenning/mini_trainer.git +``` + +In a checkout of `QUANT_SHA`, export its lockfile with an explicit CUDA backend. +Choose `cu126`, `cu130` or `cu132` according to the allocated GPU and driver; do +not choose CPU wheels. `cu130` below is an explicit example, not GPU qualification. +Keep the NVIDIA driver information from `nvidia-smi` with your results. + +```bash +uv export --locked --no-dev --no-emit-project --extra recommended --extra cu130 --extra export --extra quantization --output-file /work/requirements-mt.txt +uv venv --python 3.12 /work/venvs/mt-master +uv venv --python 3.12 /work/venvs/mt-quant +export UV_LINK_MODE=copy +unset UV_TORCH_BACKEND +uv pip install --python /work/venvs/mt-master/bin/python --index https://pypi.org/simple --default-index https://download.pytorch.org/whl/cu130 --index-strategy unsafe-first-match --require-hashes -r /work/requirements-mt.txt +uv pip install --python /work/venvs/mt-quant/bin/python --index https://pypi.org/simple --default-index https://download.pytorch.org/whl/cu130 --index-strategy unsafe-first-match --require-hashes -r /work/requirements-mt.txt +uv pip install --python /work/venvs/mt-master/bin/python --no-deps "mini_trainer @ git+$REPO_URL@$MASTER_SHA" +uv pip install --python /work/venvs/mt-quant/bin/python --no-deps "mini_trainer @ git+$REPO_URL@$QUANT_SHA" +uv pip check --python /work/venvs/mt-master/bin/python +uv pip check --python /work/venvs/mt-quant/bin/python +``` + +Both pinned commits must be available in the Git repository you install from. +At preparation time the public `quant` tip was `f5c69e7cab2bfde8a5467026b293858b93e628f9`; +the pinned local tip contains two later commits. Publish those commits before +using the HTTPS installation above, or transfer a Git bundle without publishing: + +```bash +# On the development machine; transfer this bundle and dev/ucloud to UCloud. +git bundle create /tmp/mini-trainer-ucloud.bundle master quant +# On UCloud, after transferring it: +git clone /work/mini-trainer-ucloud.bundle /work/mini-trainer-source +REPO_URL=file:///work/mini-trainer-source +# Then use the same pinned uv pip install commands above with this REPO_URL. +``` + +Keep the harness separately: it was added after the pinned quant commit and is +not installed by `uv pip install mini_trainer`. Use a checkout containing +`dev/ucloud/launch.sh`, not a checkout reset to `QUANT_SHA`, for the copy below. +Save the exported requirements. +The export and quantization extras are installed equally in both environments so +optional follow-ups do not change the dependency comparison. +Use the same backend in the export's `--extra` and the installs' explicit CUDA +index URL. Exported requirements do not carry the custom PyTorch index. +PyPI is searched first, then the CUDA index when the pinned version is absent. +Keep hashes enabled. Avoid `--torch-backend` for this exported lock: it also +redirects `torchao`, which is locked from PyPI, and can trigger an unpinned +requirement error in a fresh environment. Stop on any installation failure before +continuing to the next command. The copy link mode avoids cross-filesystem +hardlink warnings; it does not affect training speed. + +Preparation downloads torchvision's pretrained weights if absent. For offline +jobs, populate a shared writable `TORCH_HOME` ahead of time using the same locked +torchvision version. Allow CPU RAM/disk for the full hierarchical initial models, +three seeds, optimizer checkpoints and compiled caches. Do not assume INT8 makes +the convolutional backbone or all optimizer state integer. + +## Configure and launch + +If you cloned a branch containing this harness directly onto the node, run from +that checkout's `dev/ucloud` directory. No `/work/comparison` directory or copy is +needed. Keep the checkout there; the launcher resolves its sibling scripts itself +and runs workers outside the checkout with the configured installed interpreters. +For example: + +```bash +cd /work/mini_trainer/dev/ucloud +``` + +The following copy instructions are an alternative for transferring just the harness +when environments already exist. `setup.sh` requires a Git checkout with the pinned +package commits and is intended for the fresh-job workflow above. +First copy the harness and example configuration onto the node. If a checkout +containing `dev/ucloud` is already on the node, run from that checkout's root: + +```bash +mkdir -p /work/comparison +cp -i dev/ucloud/launch.sh dev/ucloud/compare.py dev/ucloud/worker.py \ + dev/ucloud/export_followup.py dev/ucloud/comparison.json dev/ucloud/qualification.json /work/comparison/ +``` + +Otherwise, run this from the checkout root on your development machine, replacing +`UCLOUD_SSH_HOST` with the SSH destination you use for the allocated node (and +adding your usual SSH port/key options if needed): + +```bash +ssh UCLOUD_SSH_HOST 'mkdir -p /work/comparison' +scp dev/ucloud/launch.sh dev/ucloud/compare.py dev/ucloud/worker.py \ + dev/ucloud/export_followup.py dev/ucloud/comparison.json dev/ucloud/qualification.json UCLOUD_SSH_HOST:/work/comparison/ +``` + +Copy once before configuring; preserve an already edited node configuration when +updating scripts. Creating `/work/comparison` alone does not populate it. Keep the +three Python scripts alongside `launch.sh`, which resolves them relative to itself. + +On the node, edit `/work/comparison/comparison.json`: set `parquet`, `output`, both +Python paths and `gpus`. +The Parquet must sit beside `images//` as expected by +`mini_trainer`. Preparation checks every image path, rejects duplicates and missing +labels, freezes the taxonomy/index, and retains the existing `set` mapping: +`0=test`, `1=validation`, other numeric sets=train. Non-numeric sets are excluded by +the existing parser. No random re-splitting occurs. Keep image bytes immutable; +preparation checks file existence, not hashes or complete decoding of the images. + +```bash +cd /work/comparison +bash launch.sh comparison.json --stage plan +bash launch.sh comparison.json --stage prepare +bash launch.sh comparison.json --stage train +bash launch.sh comparison.json --stage summary +``` + +`prepare` requires a **new** output directory and verifies CUDA in both environments. +It is CPU-heavy when parsing and initializing models. Start with the bounded +qualification profile below: one epoch of the full six-million-image dataset is +still a long run. Eight workers per rank can mean 64 workers on eight GPUs; reduce +consistently if the allocated CPU or shared-memory budget requires it. + +### Bounded qualification on one MIG device + +Use `qualification.json` for the first setup test. Edit its dataset path and +environment entries to match your node, and choose a new output path. It selects +one visible CUDA device, batch size 32, two loader workers, one epoch and one seed, +with both eager controls. The source splits are preserved, and a fixed seed selects +2,048 training, 512 validation and 128 test rows by uniform reservoir sampling within +each split. Both branches use exactly the same saved sample and starting weights. +Test images are checked for existence but are not used in training or validation. + +This gives **64 training batches and 16 validation batches per branch**. Selection +still streams the source Parquet once, but retains only 2,688 rows plus one batch +in memory. Only selected image paths are checked. The taxonomy and classifier head +are built from the selected rows, reducing the pinned master's expensive normalized +head initialization. Duplicate/path validation covers the sample, not the entire +source dataset. The selected Parquet, mappings, index and seed are retained and +hashed with the other preparation artifacts. + +This is an infrastructure smoke test, **not a full-taxonomy performance or quality +comparison**. Random sampling can leave validation species absent from the training +sample. Do not interpret its accuracy, loss weighting or timing as representative +of the complete dataset. Results carry `scope=qualification_subset`, including in +the paired report. Omitting `qualification` preserves the full-dataset workflow. + +```bash +# From the node's checkout, after editing qualification.json: +cd /work/mini_trainer/dev/ucloud +bash launch.sh qualification.json --stage plan +bash launch.sh qualification.json --stage prepare && \ + bash launch.sh qualification.json --stage train +bash launch.sh qualification.json --stage summary +``` + +The profile's `budget_seconds: 1800` sets one wall-clock deadline beginning at +`prepare`, shared by preflights, preparation and both training runs. Time between +commands also counts; chain preparation and training as above. The launcher stops +active workers when the deadline expires (cleanup may take up to 15 additional +seconds), records an interrupted/timed-out run when applicable, and preserves all +logs. This is a time bound, not a promise that both runs finish on every allocation. +`timeout_seconds` additionally caps each worker call. Summary remains available +after the deadline. The budget does not stop the UCloud allocation or include manual +environment installation; set the job lifetime in UCloud separately. + +After the eager pair passes, use another new output and add `quant_prefetch` first. +Test compilation variants separately: cold compilation can consume much of a short +budget. For timing comparisons, increase to at least two epochs to distinguish +first-use costs from subsequent execution, and keep sample seed, batch size, +image size, resource allocation and thread settings fixed across variants. Full +taxonomy, full-data convergence, INT8 and multi-GPU/DDP qualification remain separate. + +### Job resources and bottlenecks + +[UCloud's resource guide](https://docs.cloud.sdu.dk/guide/resources-products.html) +describes one MIG as one seventh of a B200 and allows one to four MIGs per job. +The UI's `4/7` selection provides four separate `1g.23gb` devices, not a single +device with four times the memory. CPU and system RAM allocations scale with the +selected product. Use the vCPU/RAM values shown for that selection; `nproc` and +`free` can expose host totals (384 vCPUs and roughly 2.2 TiB on this node type). +Keep the existing one-MIG allocation for the initial eager test; four devices +would additionally exercise DDP and would not accelerate the serial preparation. + +The profile's batch size 32 leaves more headroom than the observed full-head +batch-64 run, which used about 16 GiB of a 20.5 GiB device. Keep 384-pixel inputs to +exercise the intended preprocessing. Two loader workers are a conservative starting +point for a fractional allocation. The reported 34 GiB `/dev/shm` is not a small +default shared-memory mount; there is no evidence so far that it needs increasing. +Avoid full-dataset RAM caching. If loaders later starve the GPU, compare two and +four workers within the actual CPU quota before increasing further; keep the same +worker count for paired branches. Container CPU quotas may be lower than affinity. + +The launcher defaults to one CPU math thread and one compiler worker per rank. +Changing `OMP_NUM_THREADS` and `MKL_NUM_THREADS` before launch affects both branches; +two math threads can be tested if the allocated CPU budget leaves room alongside +the loaders. Do not set them from the host CPU count. Preflight records the actual +PyTorch thread count, visible CPU affinity count and shared-memory capacity. Keep +the pretrained weight cache (`TORCH_HOME`) on persistent writable storage to avoid +repeated downloads. If file I/O remains dominant, stage only the small sample's +images on confirmed fast node-local storage for a separate, identically staged +comparison; moving the full dataset is unnecessary for a smoke test. + +### Preparation progress and interrupted runs + +The launcher prints the preflight and preparation log paths. `prepare.log` includes +timestamped sampling, taxonomy, path-checking, index-writing, hashing and model +construction messages, with periodic counts while scanning and checking paths. +CPU model construction can still be silent inside the pinned package's initializer. +JSON is written incrementally, and per-image metadata is released before building +starting models to reduce preparation's temporary memory requirements. + +```bash +output_dir=$(python3 -c 'import json; print(json.load(open("qualification.json"))["output"])') +tail -f "$output_dir/prepare.log" +``` + +`prepared.json` is the completion marker. Its absence can mean preparation is still +running; check the original launcher and worker processes before starting another +attempt. An existing output cannot be prepared again, and training now reports an +explicit incomplete-preparation error. Preserve an interrupted attempt and use a +new output path. Ctrl-C stops the worker process group and records `interrupted` +for an active training run; it exits with code 130 without a Python traceback. +Neither interrupted nor timed-out runs enter paired completed-run comparisons. + +`--only quant_eager_seed42` selects a planned run. Run `train` again to skip +completed, checksum-verified runs and continue pending ones. Failed/interrupted +runs are preserved and block reuse of their names: use a new output directory for +a repeated comparison. Do not mix partial/resumed timings into a complete-run row. +The timeout is per run (default 32 hours), not a node reservation. Set the UCloud +job duration for preparation plus all runs. Set `CUDA_VISIBLE_DEVICES` to the +allocated devices if needed; the launcher does not overwrite it. + +The launcher defaults to one CPU math thread and one compiler worker per rank. +Set `TORCH_HOME` and, if +needed, `TMPDIR` to sufficiently large writable volumes before launching. Compiler +caches are isolated per run/rank so first-use compilation is charged to that run; +filesystem page cache and hardware temperature are not reset between runs. +Compiler caches default to unique directories in `/tmp`, recorded per rank in the +run directory. Set `MT_COMPILER_CACHE_ROOT` to a larger writable directory if +needed; it must contain no whitespace because the C++ compiler toolchain can +misparse cache paths. Dataset and result paths may contain spaces. Even eager +model runs can compile existing augmentation kernels. Remove recorded cache +directories after the experiment if their disk space is needed. +Training output is redirected to the printed `console.log` path; use `tail -f` +from a second SSH session to follow it. + +## Results and interpretation + +`comparison.csv` contains wall time, total training-phase time, first and later +epoch times, maximum per-rank allocated CUDA memory and best validation selection +metric. `paired.json` gives wall speedup and validation differences against the +same-seed master control. Retain each seed; three seeds support an initial paired +comparison, not a precise general performance claim. Compare acceleration rows +against `quant_eager` as well to isolate optional-feature effects. + +Each run retains `console.log`, `launch.json`, `result.json`, per-rank phase JSONL, +and `model/` with the trainer's configuration, summaries and `weights/best.pt` plus +training checkpoints. Wall time includes startup, loading, compilation, evaluation, +logging and saving; preparation is outside that measurement. Synchronized phase +timings include logger work. The harness disables the logger's per-step CUDA peak +reset so phase peaks cover all steps. These peaks exclude earlier model/optimizer +construction; use UCloud's resource report or `nvidia-smi` for whole-job/process +memory. CUDA allocator bytes are not total device memory or node RAM. + +The built-in selection metric is the trainer's leaf-level validation accuracy; +DDP's padded validation sampler can repeat a few examples. It is not a full +macro/per-class quality study. Test data is frozen but never used by this training +runner. Choose configurations using validation first, then perform a separate, +non-distributed held-out evaluation for per-level accuracy, rare-class recall, +probability quality and uncertainty. Preserve failed and slower configurations. + +## ONNX follow-up using these checkpoints + +After training, use the **quant environment** to export a selected floating-point +checkpoint (including a master-trained checkpoint). This keeps the deployment +toolchain fixed while comparing learned weights: + +```bash +/work/venvs/mt-quant/bin/python /work/comparison/export_followup.py /work/results/global-lepi-ddp quant_eager_seed42 +``` + +This exports the actual hierarchical evaluation forward, runs dynamic-batch parity, +and compares CPU ONNX Runtime with reloaded FP32 PyTorch on four real validation +images through the checkpoint preprocessor. It retains the complete ONNX bundle, +source checkpoint hash, preprocessing description, image hashes, and +`validation-example.npz`. The destination must be new. It does not rerun training +or access test images. + +A standalone environment with NumPy and ONNX Runtime can replay the saved inputs: + +```python +import json +import numpy as np +import onnxruntime as ort + +bundle = "/work/results/global-lepi-ddp/runs/quant_eager_seed42/onnx" +with open(f"{bundle}/manifest.json") as handle: + manifest = json.load(handle) +session = ort.InferenceSession(f"{bundle}/model.onnx", providers=["CPUExecutionProvider"]) +with np.load(f"{bundle}/validation-example.npz") as example: + outputs = session.run(None, {manifest["input"]["name"]: example["images"]}) + for i, output in enumerate(outputs): + np.testing.assert_allclose(output, example[f"output_{i}"], rtol=1e-4, atol=1e-5) +``` + +This establishes a small real-image export/inference parity check, not target-GPU +latency, dataset-wide quality or a standalone image decoder. Follow the +[ONNX guide](../../docs/onnx.md) and existing +[deployment benchmark workflow](../benchmarks/inference.md) for provider placement, +warmup/repeated timing, held-out inference and INT8 export. Native INT8 checkpoints +require their explicit CUDA-reference export path and are rejected by this FP32 +follow-up. Calibrate PTQ on training images only. + +## Local validation and limits + +The bounded qualification increment has 16 focused passing cases covering the +bootstrap's success/failure flow with a fake installer, reproducible split sampling, +CPU preparation/training/reload with real EfficientNetV2-S, artifact checks, +interruption records, and deadline termination of a real worker. The core harness +tests also passed against the pinned master source. The bootstrap's real locked +export and dependency-sync dry run resolved the Python 3.12 CUDA 13.0 environment; +a complete fresh GPU environment was not installed locally. + +Static checks passed. The repository-wide run completed its benchmark group, then +stalled in a spawned loader test in the sandbox; the remaining suite was rerun +outside the sandbox with one CPU math thread: 377 passed, 149 skipped, and the known +EMA expected failure. Skips include unavailable GPU/optional/slow coverage. Warnings +include upstream deprecations and missing optional dendrogram plotting dependencies. +There is no new B200/MIG runtime measurement for the bounded profile yet. + +Initial harness validation, before the bounded profile: + +The launch planner, input freezing, completed-run skipping, shell syntax, Ruff and +import contracts were checked. A small Parquet fixture ran through preparation, +one CPU training epoch and checkpoint reload with the real EfficientNetV2-S +hierarchical model on both the current quant code and pinned master source, +without downloading pretrained weights. The ONNX follow-up passed dynamic-batch +and four-image CPU Runtime parity using the resulting quant checkpoint. + +Focused harness, training-state, integration and ONNX checks passed: 108 tests +across the validation runs, 43 hardware/optional-dependency skips and the known EMA +expected failure. CPU DDP passed when rerun outside the sandbox's socket +restriction. The repository-wide suite was interrupted in an unrelated long-running +benchmark; it was not completed. No GPU was available locally. These checks do not +establish CUDA/DDP compilation, INT8 correctness or global_lepi convergence and +throughput on UCloud; run the qualification configuration there first. + +## Expanded single-GPU experiment and validation-loss diagnosis + +`experiment.json` increases the qualification to 4,096 train / 1,024 validation / +128 test images and four epochs. Existing split assignments remain intact; the +sample builds its own taxonomy, so this still does not measure full-head quality. +It keeps batch 32, two loader workers, 384 pixels, seed 42 and the same pinned +packages. The three variants are master eager, quant eager and quant prefetch. +Prefetch remains diagnostic after an observed first-epoch validation NaN; finite +training losses do not establish that validation is numerically sound. + +There are 128 training batches and 32 validation batches per epoch. Compare the +mean of epochs 2–4 (`later_epoch_mean_seconds`), and retain first-epoch and total +wall times separately. The expanded run measured about 31 seconds for the first +training epoch and 25.5 seconds for later epochs, but its first validation phase +(including figures) took 151 seconds and the run exceeded five minutes. +The template now sets `figures: false` for every variant, bypassing confusion, +class-distance and dendrogram figures (including species-name lookups). Scalar +validation metrics, losses, checkpoint selection and saving still run. Existing +configs default to figures enabled. This setting only changes the dedicated +harness worker; no package reinstall is required. Allow roughly 2–3 minutes per +variant based on those training times, with additional time possible for cold +image reads and startup. Whole-run and validation-phase timings are not directly +comparable to older runs with figures enabled. +Each worker has a 300-second limit; timeout terminates the experiment rather than +silently shortening its epochs. Process cleanup can take another 15 seconds. +The overall prepare/train budget is 1,800 seconds, including pauses between stages. +A fresh job is unnecessary; pull the harness update and use a new output directory. + +`require_finite_losses: true` audits all expected training and validation epoch +rows in `model/logs/summary.csv`. Missing, malformed or non-finite total/per-level +losses produce `status=invalid_metrics` and `loss_check=failed`, even with exit code +zero. `result.json` identifies the affected epoch and phase. Checkpoints and timing +artifacts remain available, but such runs are excluded from successful paired +comparisons. This is a recorded-loss check, not a claim that the optimizer failed, +nor a guarantee that every intermediate tensor was finite. Existing configs retain +their previous behavior unless the flag is enabled. + +First replay the affected checkpoint in four fresh processes. This uses the frozen +validation images, training class frequencies, smoothing and saved preprocessing. +FP16 means FP32 model parameters with FP16 autocast; FP32 disables autocast and +builds the criterion in FP32. No optimizer, augmentation or training runs. Each +case records parameter finiteness, per-batch input/output/loss finiteness and input +and label hashes under a new output directory. It never rewrites the old run. +Use the pinned quant interpreter, with the same thread settings as the launcher: + +```bash +cd /work/mini_trainer +git pull --ff-only +export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 +for precision in fp16 fp32; do + for loader in eager prefetch; do + replay_args=() + if [[ "$loader" == prefetch ]]; then replay_args+=(--prefetch); fi + timeout --kill-after=15s 300s /work/venvs/mt-quant/bin/python \ + dev/ucloud/replay_validation.py \ + /work/results/global-lepi-prefetch-1 quant_prefetch_seed42 \ + --precision "$precision" "${replay_args[@]}" \ + --output "/work/results/prefetch-replay-1-$precision-$loader" || break 2 + done +done +``` + +A completed diagnostic writes `result.json` even when it finds non-finite values; +inspect all four reports and `batches.jsonl`. Matching input/target hashes help +check that cases consumed identical batches. FP16-only failure points toward a +precision-dependent evaluation issue; prefetch-only failure warrants transfer-path +investigation. Neither outcome alone proves the root cause. Reloading clears +in-memory model caches, and per-batch checks synchronize CUDA, so a clean replay +cannot rule out a timing-sensitive failure in the original process. This is not a +speed benchmark. The original NaN occurred during evaluation, not recorded training +loss, and does not by itself demonstrate training divergence. + +Then generate the expanded config, preserving this job's dataset and environment +paths. It intentionally creates a fresh preparation and starting weights: + +```bash +python3 - <<'PY' +import json +from pathlib import Path +config = json.loads(Path('dev/ucloud/experiment.json').read_text()) +previous = json.loads(Path('/work/qualification-prefetch.json').read_text()) +for key in ('parquet', 'environments'): + config[key] = previous[key] +with Path('/work/qualification-expanded.json').open('x') as handle: + json.dump(config, handle, indent=2) + handle.write('\n') +PY +bash dev/ucloud/launch.sh /work/qualification-expanded.json --stage plan +bash dev/ucloud/launch.sh /work/qualification-expanded.json --stage prepare && \ + bash dev/ucloud/launch.sh /work/qualification-expanded.json --stage train +bash dev/ucloud/launch.sh /work/qualification-expanded.json --stage summary +cat /work/results/global-lepi-expanded-nofigures-1/comparison.csv +``` + +Run in tmux and retain all artifacts, including failed validation checks. A warm +first epoch is kept in the results, not discarded from model training; the later +three epochs provide the timing comparison. This is a larger exploratory test +with one paired seed, not a statistical quality comparison. Do not interpret +small accuracy or timing differences as established improvements. If the replay +finds non-finite values, keep prefetch results diagnostic while investigating them; +eager controls remain useful. Compiler variants are a subsequent experiment. + +## Model compilation comparison in the existing job + +Use `compilation.json` after the expanded eager/prefetch experiment. It keeps +4,096 training / 1,024 validation / 128 test images, four epochs, seed 42, +batch 32, two workers, 384 pixels, figures disabled and the same pinned venvs. +It replaces prefetch with `quant_compile_model` (model compilation only). +The deterministic run order is quant eager, master eager, then quant model +compilation, so both controls finish before a possible compilation timeout. +No dedicated NaN tracing or replay is required for this experiment. The existing +recorded-loss audit remains enabled; `invalid_metrics` retains timing evidence +but excludes that run from the automatic successful-pair report. + +Pull into the current job and run in tmux. The template uses the dataset and venv +paths from the existing job and a fresh output directory; no installation is +needed. If those paths differ, copy the JSON and edit it before preparation. +Do not reuse or modify an already prepared output directory. + +```bash +cd /work/mini_trainer +git pull --ff-only +bash dev/ucloud/launch.sh dev/ucloud/compilation.json --stage plan +bash dev/ucloud/launch.sh dev/ucloud/compilation.json --stage prepare && \ + bash dev/ucloud/launch.sh dev/ucloud/compilation.json --stage train +# Run summary even if training reports a timeout or invalid metrics. +bash dev/ucloud/launch.sh dev/ucloud/compilation.json --stage summary +cat /work/results/global-lepi-compile-model-1/comparison.csv +cat /work/results/global-lepi-compile-model-1/paired.json +``` + +Each worker has a 300-second limit, with up to 15 seconds for termination cleanup. +The 1,800-second overall budget includes preparation and gaps between commands. +Based on the existing eager runs, allow about five minutes for the two controls +combined, up to five minutes for compilation/training, plus preparation. This is +an estimate, not a promise that the compiled variant will finish. + +The harness creates fresh compiler cache directories for each run. No separate +compiled warmup is performed: cold compilation cost belongs in `wall_seconds` +and the first epoch. Compare those with `later_epoch_mean_seconds` (epochs 2–4), +`train_seconds`, and `peak_allocated_bytes`. Later epochs may still include +recompilation; retain the console log when interpreting them. Compare primarily +against quant eager to isolate the effect of compilation, with master eager as +the branch control. A cold-start timeout establishes that this configuration +does not fit the short-job budget; it does not rule out longer-run benefits. +Small single-seed timing differences are exploratory, not established speedups. + +## Optimizer compilation comparison in the existing job + +`optimizer-compilation.json` tests optimizer compilation alone. It keeps the model +eager, prefetch disabled and optimizer CUDA graphs disabled. The workload remains +4,096 train / 1,024 validation / 128 test images, four epochs, seed 42, batch 32, +two loader workers and 384 pixels on one GPU. Figures are disabled and the +recorded-loss audit stays enabled. Installed master and quant revisions are unchanged. + +The seeded order is quant eager, master eager, then `quant_compile_optimizer`. +Both controls therefore run before a possible compiler timeout. Each worker is +limited to 300 seconds (plus up to 15 seconds cleanup), and the overall budget is +1,800 seconds including preparation and pauses between stages. Allow roughly +10–12 minutes including preparation, based on the previous eager timings and +the compiled worker's limit; this does not guarantee compilation will finish. + +Run these commands in tmux in the current allocated job. No venv rebuild is +needed. The new output directory preserves all earlier experiment artifacts. + +```bash +cd /work/mini_trainer +git pull --ff-only +bash dev/ucloud/launch.sh dev/ucloud/optimizer-compilation.json --stage plan +bash dev/ucloud/launch.sh dev/ucloud/optimizer-compilation.json --stage prepare && \ + bash dev/ucloud/launch.sh dev/ucloud/optimizer-compilation.json --stage train +# Run summary even after a timeout or an invalid-metrics report. +bash dev/ucloud/launch.sh dev/ucloud/optimizer-compilation.json --stage summary +cat /work/results/global-lepi-compile-optimizer-1/comparison.csv +cat /work/results/global-lepi-compile-optimizer-1/paired.json +``` + +Monitor the compiled run from a second terminal: + +```bash +tail -F /work/results/global-lepi-compile-optimizer-1/runs/quant_compile_optimizer_seed42/console.log +``` + +Compare the compiled optimizer primarily against quant eager within this run: +`wall_seconds`, `first_epoch_seconds`, `later_epoch_mean_seconds` and +`peak_allocated_bytes`. Compiler caches start fresh; no unmeasured compilation +warmup is added. Later epochs can still incur recompilation, so keep warnings +with the timing results. The existing optimizer, learning-rate schedule and AMP +settings remain in effect. This experiment does not combine model compilation +with optimizer compilation; it measures their effects separately first. + +If master has `invalid_metrics`, `paired.json` will be empty because that report +requires a completed master baseline. The CSV still contains the direct quant +comparison. Preserve the loss-audit flag alongside exploratory timing conclusions; +no dedicated numerical tracing is a prerequisite for this experiment. + +## Combined compilation, prefetch and native INT8 + +`combined-int8.json` compares the following runs in this seeded execution order: + +| Variant | Installed branch | Native INT8 | Model compile | Optimizer compile | Prefetch | +| --- | --- | --- | --- | --- | --- | +| `quant_eager` | quant | No | No | No | No | +| `master_eager` | master | No | No | No | No | +| `quant_compile_model` | quant | No | Yes | No | No | +| `quant_compile_both` | quant | No | Yes | Yes | No | +| `quant_float_combined` | quant | No | Yes | Yes | Yes | +| `quant_int8_combined` | quant | Yes | Yes | Yes | Yes | + +Both combined variants use the same pinned quant package. Here “float” means +ordinary FP32 parameters with FP16 AMP, not a switch to FP32-only training. +Native INT8 applies to eligible linear modules; convolutions, normalization, +biases and other unsupported operations remain floating point. Check the +`INT8 training coverage:` entry in the INT8 console log for the actual coverage. +This is not an entirely INT8 EfficientNet. Optimizer CUDA graphs remain disabled. +The older `quant_combined` name remains a floating-point variant for compatibility. + +The four-epoch, 4,096/1,024/128-image workload, batch 32, two workers, 384 pixels, +seed 42, figures disabled and loss audit enabled are unchanged. Each worker has +a 300-second timeout (plus up to 15 seconds cleanup). The entire experiment has +a 1,800-second budget including preparation and gaps between stages. Six workers +all reaching their limits would exceed that budget, so the controller may stop +before the last run in that case. Based on previous timings, allow approximately +20–25 minutes; INT8 and combined compilation are unverified on this allocation. +Fresh compiler caches keep compilation cost in each run. No numerical tracing +or extra warmup is required. + +Pull into the current job and run in tmux; existing venvs already include the +quantization extra. Preparation checks the dependency and single-GPU restriction +for any variant requesting INT8, including the new combined variant. + +```bash +cd /work/mini_trainer +git pull --ff-only +bash dev/ucloud/launch.sh dev/ucloud/combined-int8.json --stage plan +bash dev/ucloud/launch.sh dev/ucloud/combined-int8.json --stage prepare && \ + bash dev/ucloud/launch.sh dev/ucloud/combined-int8.json --stage train +# Run even if training reports a failure, timeout or invalid metrics. +bash dev/ucloud/launch.sh dev/ucloud/combined-int8.json --stage summary +cat /work/results/global-lepi-combined-int8-1/comparison.csv +cat /work/results/global-lepi-combined-int8-1/paired.json +``` + +The controller continues after ordinary nonzero worker exits or invalid metrics, +but stops on a timeout. Preserve the logs. If the floating combined worker times +out and the overall budget has time left, launch just the not-yet-started INT8 run: + +```bash +bash dev/ucloud/launch.sh dev/ucloud/combined-int8.json --stage train \ + --only quant_int8_combined_seed42 +bash dev/ucloud/launch.sh dev/ucloud/combined-int8.json --stage summary +``` + +Do not rerun an existing failed directory or reset the experiment deadline. +Compare model-only against both compilations to test their interaction, then +both compilations against float combined to test prefetch, then float combined +against INT8 combined to test quantized training. Use total wall time, first-epoch +time, later-epoch mean and peak allocated memory, retaining loss-audit status. +The automatic paired report uses master; the CSV supports these direct quant +comparisons even when master is excluded by its loss audit. One seed does not +establish quality equivalence or a small performance improvement. + +## Figures-enabled qualification + +`figures.json` exercises the simplified dendrogram renderer, compact SVG export, +and bounded whole-matrix confusion reporting for two +epochs on the existing 4,096/1,024/128-image subset. Use only +`quant_compile_model_seed42`: floating-point training with model compilation, +without optimizer compilation or prefetch. The eager variants remain in the +configuration to satisfy harness validation; they are not selected below. +The worker retains its 300-second timeout and the experiment its 1,800-second +budget. Cold compilation and taxonomy lookups are included; completion within +five minutes needs confirmation on the allocated job. + +This fix changes the installed package, so pull the harness and explicitly +upgrade the quant venv to the code revision pinned in this config. Existing +benchmark configs retain their older pins and will need a matching environment +if reused. Run in tmux: + +```bash +cd /work/mini_trainer +git pull --ff-only +FIGURES_SHA=$(python3 -c 'import json; print(json.load(open("dev/ucloud/figures.json"))["environments"]["quant"]["commit"])') +uv pip install --python /work/venvs/mt-quant/bin/python --no-deps --link-mode=copy \ + "mini_trainer @ git+https://github.com/asgersvenning/mini_trainer.git@$FIGURES_SHA" +uv pip check --python /work/venvs/mt-quant/bin/python +bash dev/ucloud/launch.sh dev/ucloud/figures.json --stage plan +bash dev/ucloud/launch.sh dev/ucloud/figures.json --stage prepare && \ + bash dev/ucloud/launch.sh dev/ucloud/figures.json --stage train \ + --only quant_compile_model_seed42 +# Summarize even if training reports a timeout or invalid metrics. +bash dev/ucloud/launch.sh dev/ucloud/figures.json --stage summary +cat /work/results/global-lepi-figures-3/comparison.csv +``` + +Inspect the console log's rendering/export timings and the saved figures under +`/work/results/global-lepi-figures-3/runs/quant_compile_model_seed42/model/logs/figures/`. +Both `epoch-0001` and `epoch-0002` should contain readable dendrogram SVGs, +captioned confusion overview PNGs and class-distance PNGs. The per-level +`Confusion_matrix_lvlL/` and `Soft_confusion_matrix_lvlL/` subdirectories retain +full-resolution PNGs, numerical data, class indices and color scales. Dashboard +overviews cover every class in the original order. See +[confusion reporting details and measurements](../benchmarks/reporting/confusion.md). +Compare first and second epoch reporting costs to check label-cache reuse. `paired.json` will be empty because no master run was selected; this is a +figure qualification, not a new branch comparison. Preserve any failed output +and choose a new output path for a retry. + +See [renderer measurements and limitations](../benchmarks/reporting/dendrogram.md). + +## Scaling the selected configuration + +For four full GPUs with figures, W&B, a bounded batch sweep and checkpoint +continuation, use [the DDP qualification and production handoff](ddp.md). +Production training uses `mt_htrain` under `torchrun`; the Python API harness +remains a qualification tool. + +For the post-training expert benchmark, start with the +[bounded RAM-staging inference trial](expert-trial.md). It generates its minimal +configuration and runs the standard prediction CLI without rebuilding an index. + +## Calibrate filesystem read concurrency + +`calibrate_io.py` is a standalone Python file; copying that one file is sufficient. +It uses standard-library threads and subprocesses, with Pillow needed only for +`--mode decode`. It does not import mini_trainer or alter its environment. + +```bash +/work/venvs/mt-quant/bin/python dev/ucloud/calibrate_io.py \ + /work/flemming_helsing/restructured/valid/referenced \ + --mode stage --destination /dev/shm --output /work/expert-io-calibration.json +``` + +The default sweep tests 1–1,024 concurrent reads with 30-second I/O measurement limits, +then retests the three strongest candidates twice in shuffled order. It recommends +the smallest concurrency within 5% of the best median confirmation throughput. +Every thread is initialized before measurement starts. Process/thread startup has +a separate 30-second limit and is recorded independently; the overall budget is +600 seconds. blocked filesystem metadata calls may +outlast the budget. Each trial consumes only paths actually submitted to its reader pool; untouched +paths remain available after a timeout. Submitted selections are disjoint and +shuffled. Shared cache state remains unknown. Run away from other heavy read jobs +when selecting a baseline. Existing reports are preserved and a numeric suffix is +chosen automatically for subsequent runs. + +`--mode read` (the default) measures concurrent encoded-byte reads without retaining +a second copy. `--mode stage --destination PATH` also measures writes to the intended +staging filesystem; its own temporary copies are removed between trials. +`--mode decode --resize 384` includes RGB decoding and optional CPU resize, for a +loader-oriented measurement. The resulting thread count is for concurrent I/O or +read/decode tasks, **not** a recommendation to create that many DataLoader processes. +This file calibrates concurrency; it does not install read-ahead into training. + +`--workers`, `--files-per-trial`, `--trial-seconds`, `--startup-seconds`, `--budget-seconds`, and +`--confirmation-rounds` are adjustable. Each trial selects at most 2,048 files and +16 GiB of encoded data by default. Selection shrinks automatically to fit the byte +cap and staging destination; actual and requested concurrency are both recorded +when fewer files fit. Small trials (including ten-file trials) run normally, and +completed reads are ranked without an arbitrary 32-image minimum. Linux child RSS +is monitored against a 4 GiB stop threshold. Failed settings are excluded. The JSON report contains per-trial +throughput, latency, errors, sampled paths and the recommendation. If samples or +time are exhausted, inspect the confirmation coverage before treating the result +as repeatable. Very fast RAM trials benefit from increasing `--files-per-trial`. + +Show an existing report without printing its sample manifest: + +```bash +/work/venvs/mt-quant/bin/python dev/ucloud/calibrate_io.py \ + --summary /work/io-calibration.json +``` + +Each run also writes a compact `.summary.txt` beside its JSON. Calibration failures +are recorded in the report and a separate `.error.log`, including failures to +terminate a blocked I/O subprocess. Completed partial trials remain usable; an +unusable confirmation does not erase a valid sweep estimate (marked provisional). +A setting that fails with an error or memory-limit violation during confirmation +is excluded from that fallback. An uninterruptible kernel I/O wait can delay process +termination; the calibrator stops instead of accumulating competing readers. diff --git a/dev/ucloud/calibrate_io.py b/dev/ucloud/calibrate_io.py new file mode 100644 index 00000000..1ad7c966 --- /dev/null +++ b/dev/ucloud/calibrate_io.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Standalone filesystem image-I/O concurrency calibration (Python 3.12+, POSIX). + +Examples: + python calibrate_io.py /images --mode stage --destination /dev/shm --output /work/io.json + python calibrate_io.py /images --mode decode --resize 384 --output /work/decode.json + +Read an existing report without its path manifest: + python calibrate_io.py --summary /work/io.json + +Only decode mode requires Pillow. No mini_trainer, PyTorch, root access or cache +flushing. All trials use disjoint selections; externally warmed caches are unknown. +""" + +import argparse +import json +import os +import random +import resource +import shutil +import signal +import statistics +import subprocess +import sys +import tempfile +import threading +import time +import traceback +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from pathlib import Path + +EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"} + + +def write_json(path, value): + temporary = Path(str(path) + ".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n") + temporary.replace(path) + + +def discover(source, limit, deadline, rng): + """Shuffle traversal before bounded discovery; never read source contents.""" + paths = [] + for directory, folders, files in os.walk(source): + rng.shuffle(folders) + rng.shuffle(files) + for name in files: + if Path(name).suffix.lower() in EXTENSIONS: + paths.append(str(Path(directory) / name)) + if len(paths) >= limit: + rng.shuffle(paths) + return paths, True + if time.monotonic() >= deadline: + raise TimeoutError("Budget exhausted enumerating paths") + rng.shuffle(paths) + return paths, False + + +def warm_pool(pool, workers): + """Create every thread before starting any filesystem work.""" + barrier = threading.Barrier(workers + 1) + try: + futures = [pool.submit(barrier.wait) for _ in range(workers)] + except BaseException: + barrier.abort() + raise + barrier.wait() + for future in futures: + future.result() + + +def child(job_path): + initialized = time.monotonic() + job = json.loads(Path(job_path).read_text()) + image_module = None + if job["mode"] == "decode": + from PIL import Image + + image_module = Image + started = time.monotonic() + + def operation(item): + index, source = item + began = time.monotonic() + if image_module is not None: + with image_module.open(source) as image: + converted = image.convert("RGB") + if job["resize"]: + resized = converted.resize((job["resize"], job["resize"]), image_module.Resampling.BICUBIC) + resized.close() + converted.close() + size = os.stat(source).st_size + else: + size = 0 + target = open(Path(job["destination"]) / str(index), "xb") if job["mode"] == "stage" else None + try: + with open(source, "rb") as stream: + while chunk := stream.read(1024 * 1024): + if target is not None: + target.write(chunk) + size += len(chunk) + finally: + if target is not None: + target.close() + return size, time.monotonic() - began + + rows, errors = [], [] + first_completion = None + + def snapshot(finished=False): + elapsed = time.monotonic() - started + write_json( + job["result"], + { + "completed": len(rows), + "bytes": sum(row[0] for row in rows), + "seconds": elapsed, + "images_per_second": len(rows) / elapsed, + "mib_per_second": sum(row[0] for row in rows) / 2**20 / elapsed, + "latency_median_seconds": statistics.median(row[1] for row in rows) if rows else None, + "latency_p95_seconds": sorted(row[1] for row in rows)[int((len(rows) - 1) * 0.95)] if rows else None, + "first_completion_seconds": first_completion, + "errors": errors, + "finished": finished, + "peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + }, + ) + + items = iter(enumerate(job["paths"])) + # Bound outstanding tasks and buffers instead of queuing the whole dataset. + with open(job["attempted"], "wb", buffering=0) as journal, ThreadPoolExecutor(max_workers=job["workers"]) as pool: + warm_pool(pool, job["workers"]) + started = time.monotonic() + write_json(job["ready"], {"started": started, "initialization_seconds": started - initialized}) + next_snapshot = started + 0.25 + + def submit(item): + # Record BEFORE submission. After a timeout only this prefix may + # have read source bytes; the untouched suffix remains available. + journal.write(b"x") + return pool.submit(operation, item) + + pending = {submit(item) for item in list_next(items, job["workers"])} + while pending: + done, pending = wait(pending, timeout=0.25, return_when=FIRST_COMPLETED) + for future in done: + try: + rows.append(future.result()) + if first_completion is None: + first_completion = time.monotonic() - started + except Exception as error: + errors.append(f"{type(error).__name__}: {error}") + item = next(items, None) + if item is not None and not errors: + pending.add(submit(item)) + if errors or time.monotonic() >= next_snapshot: + snapshot() + next_snapshot = time.monotonic() + 0.25 + if errors: + break + snapshot(finished=True) + + +def list_next(iterator, count): + result = [] + for _ in range(count): + value = next(iterator, None) + if value is None: + break + result.append(value) + return result + + +def stop(proc): + """Do not launch another trial while an uninterruptible reader survives.""" + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + proc.wait(timeout=2) + except subprocess.TimeoutExpired: + raise RuntimeError(f"I/O process {proc.pid} remains blocked after termination; stop calibration here") from None + + +def run_trial(args, paths, workers, phase, work, deadline): + started = time.monotonic() + byte_limit = args.max_mib * 2**20 + if args.mode == "stage": + byte_limit = min(byte_limit, max(0, shutil.disk_usage(args.destination).free - 256 * 2**20)) + sizes = [] + total = 0 + for path in paths: + size = os.stat(path).st_size + if total + size > byte_limit: + break + sizes.append(size) + total += size + if not sizes: + raise RuntimeError(f"A single image exceeds available trial space ({byte_limit / 2**20:.1f} MiB): {paths[0]}") + paths = paths[: len(sizes)] + requested_workers = workers + workers = min(workers, len(paths)) + result_path, job_path = work / "result.json", work / "job.json" + attempted_path = work / "attempted.bin" + ready_path = work / "ready.json" + ready_path.unlink(missing_ok=True) + result_path.unlink(missing_ok=True) + attempted_path.unlink(missing_ok=True) + destination = None + if args.mode == "stage": + if shutil.disk_usage(args.destination).free < sum(sizes) + 256 * 2**20: + raise RuntimeError("Staging destination has insufficient free space") + destination = tempfile.mkdtemp(prefix="mt-calibrate-", dir=args.destination) + job = { + "paths": paths, + "workers": workers, + "mode": args.mode, + "resize": args.resize, + "destination": destination, + "result": str(result_path), + "attempted": str(attempted_path), + "ready": str(ready_path), + } + write_json(job_path, job) + proc = None + reason = None + try: + with (work / "child.log").open("w") as log: + proc = subprocess.Popen( + [sys.executable, str(Path(__file__).resolve()), "--child", str(job_path)], stdout=log, stderr=log, start_new_session=True + ) + until = min(deadline, time.monotonic() + getattr(args, "startup_seconds", 30)) + ready = None + while proc.poll() is None: + if ready is None and ready_path.exists(): + ready = json.loads(ready_path.read_text()) + until = min(deadline, ready["started"] + args.trial_seconds) + if time.monotonic() >= until: + reason = "time_limit" if ready is not None else "startup_time_limit" + stop(proc) + break + # Linux exposes current RSS; final peak RSS is recorded by the child. + status = Path(f"/proc/{proc.pid}/status") + if status.exists(): + try: + rss = next((int(line.split()[1]) for line in status.read_text().splitlines() if line.startswith("VmRSS:")), 0) + except FileNotFoundError: + rss = 0 + if rss > args.max_rss_mib * 1024: + reason = "memory_limit" + stop(proc) + break + time.sleep(0.1) + if ready is None and ready_path.exists(): + ready = json.loads(ready_path.read_text()) + row = ( + json.loads(result_path.read_text()) + if result_path.exists() + else { + "completed": 0, + "images_per_second": 0, + "seconds": time.monotonic() - ready["started"] if ready is not None else 0, + "errors": [], + "finished": False, + } + ) + row.update( + workers=workers, + requested_workers=requested_workers, + initialization_seconds=ready["initialization_seconds"] if ready is not None else None, + attempted=attempted_path.stat().st_size if attempted_path.exists() else 0, + phase=phase, + selected=len(paths), + selected_bytes=sum(sizes), + termination=reason, + wall_seconds=time.monotonic() - started, + ) + if proc.returncode and reason is None: + row["errors"].append((work / "child.log").read_text()[-2000:]) + row["eligible"] = not row["errors"] and reason != "memory_limit" and row["completed"] > 0 + return row + finally: + if proc is not None and proc.poll() is None: + stop(proc) + if destination is not None: + shutil.rmtree(destination) + + +def recommend(rows, tolerance): + confirmations = [row for row in rows if row["phase"] == "confirm"] + considered = confirmations if confirmations else rows + scores = {} + for workers in sorted({row["workers"] for row in considered}): + group = [row for row in considered if row["workers"] == workers] + if group and all(row["eligible"] for row in group): + scores[workers] = statistics.median(row["images_per_second"] for row in group) + if not scores: + if confirmations: + failed = {row["workers"] for row in confirmations if row.get("errors") or row.get("termination") == "memory_limit"} + provisional = recommend([row for row in rows if row["phase"] == "sweep" and row["workers"] not in failed], tolerance) + provisional["confirmation_note"] = "No usable confirmation; showing sweep estimate only" + return provisional + return {"workers": None, "reason": "No eligible trials"} + best = max(scores.values()) + selected = min(workers for workers, rate in scores.items() if rate >= best * (1 - tolerance)) + return { + "workers": selected, + "median_images_per_second": scores[selected], + "scores": scores, + "basis": "confirmation" if confirmations else "provisional_sweep", + "near_best_tolerance": tolerance, + } + + +def summary(report): + lines = [f"Mode: {report.get('mode', 'unknown')} | Source: {report.get('source', 'unknown')}"] + for row in report.get("trials", []): + lines.append( + f"{row['phase']:7} {row['workers']:4} threads | {row['images_per_second']:.1f} images/s | " + f"{row['completed']} complete | {row.get('termination') or 'finished'}" + ) + if row.get("errors"): + lines.append(" error: " + str(row["errors"][-1]).strip().split("\n")[-1][:200]) + lines.append("Recommendation: " + json.dumps(report.get("recommendation"))) + if report.get("error"): + lines.append("Failure: " + report["error"]) + return "\n".join(lines) + "\n" + + +def main(): + if len(sys.argv) == 3 and sys.argv[1] == "--summary": + print(summary(json.loads(Path(sys.argv[2]).read_text())), end="") + return + if len(sys.argv) == 3 and sys.argv[1] == "--child": + child(sys.argv[2]) + return + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("source", type=Path) + parser.add_argument("--mode", choices=("read", "stage", "decode"), default="read") + parser.add_argument("--destination", type=Path, help="Required for stage; use the intended real staging filesystem") + parser.add_argument("--output", type=Path, default=Path("io-calibration.json")) + parser.add_argument("--workers", default="1,4,16,64,128,256,512,1024") + parser.add_argument("--files-per-trial", type=int, default=2048, help="Maximum per trial; only submitted paths are consumed") + parser.add_argument("--trial-seconds", type=float, default=30, help="I/O measurement time, excluding initialization") + parser.add_argument("--startup-seconds", type=float, default=30, help="Separate process/thread initialization limit") + parser.add_argument("--budget-seconds", type=float, default=600) + parser.add_argument("--confirmation-rounds", type=int, default=2) + parser.add_argument("--max-mib", type=int, default=16384, help="Encoded byte cap per trial; sample size shrinks automatically") + parser.add_argument("--max-rss-mib", type=int, default=4096, help="Child RSS stop threshold on Linux") + parser.add_argument("--resize", type=int, default=0, help="Optional square bicubic resize in decode mode") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--tolerance", type=float, default=0.05, help="Prefer lowest concurrency within this fraction of best") + args = parser.parse_args() + workers = sorted(set(int(value) for value in args.workers.split(","))) + if ( + not workers + or min(workers) < 1 + or min(args.files_per_trial, args.trial_seconds, args.startup_seconds, args.budget_seconds, args.max_mib, args.max_rss_mib) <= 0 + ): + parser.error("Concurrency and limits must be positive") + if args.confirmation_rounds < 0 or args.resize < 0 or not 0 <= args.tolerance < 1: + parser.error("Invalid confirmation rounds, resize or tolerance") + args.source = args.source.resolve(strict=True) + if args.mode == "stage": + if args.destination is None: + parser.error("--destination is required for stage mode") + args.destination = args.destination.resolve(strict=True) + if args.destination.is_relative_to(args.source): + parser.error("Staging destination must be outside the source tree") + original_output = args.output + suffix = 2 + while args.output.exists(): + args.output = original_output.with_name(f"{original_output.stem}-{suffix}{original_output.suffix}") + suffix += 1 + print(f"Report: {args.output}", flush=True) + args.output.parent.mkdir(parents=True, exist_ok=True) + if max(workers) > args.files_per_trial: + workers = sorted({min(value, args.files_per_trial) for value in workers}) + print(f"Concurrency capped by --files-per-trial: {workers}", flush=True) + rng = random.Random(args.seed) + deadline = time.monotonic() + args.budget_seconds + maximum = args.files_per_trial * (len(workers) + 3 * args.confirmation_rounds) + print(f"Discovering up to {maximum} paths without reading image contents...", flush=True) + paths, limited = discover(args.source, maximum, deadline, rng) + report = { + "mode": args.mode, + "source": str(args.source), + "destination": str(args.destination), + "seed": args.seed, + "cache_state": "unknown; disjoint selections, no cache flushing", + "discovery_limited": limited, + "discovered": len(paths), + "trials": [], + "selection_paths": [], + "recommendation": None, + } + print(f"Found {len(paths)} candidates. Each path is assigned to at most one trial.", flush=True) + cursor = 0 + + def measure(concurrency, phase, work): + nonlocal cursor + count = min(args.files_per_trial, len(paths) - cursor) + if count == 0 or time.monotonic() >= deadline: + return False + selected = paths[cursor : cursor + count] + try: + row = run_trial(args, selected, concurrency, phase, work, deadline) + except BaseException: + journal = work / "attempted.bin" + attempted = journal.stat().st_size if journal.exists() else 0 + report["failed_trial"] = {"workers": concurrency, "phase": phase, "attempted": attempted} + report["selection_paths"].append(selected[:attempted]) + cursor += attempted + raise + cursor += row["attempted"] + report["trials"].append(row) + report["selection_paths"].append(selected[: row["attempted"]]) + report["recommendation"] = recommend(report["trials"], args.tolerance) + write_json(args.output, report) + print( + f"{phase:7} {row['workers']:4} threads (requested {concurrency}): {row['images_per_second']:9.1f} images/s; " + f"{row['completed']} completed, {row['attempted']} attempted; {row['termination'] or 'finished'}", + flush=True, + ) + return True + + try: + with tempfile.TemporaryDirectory(prefix="mt-io-calibration-") as temporary: + work = Path(temporary) + for concurrency in workers: + if not measure(concurrency, "sweep", work): + break + top = sorted((row for row in report["trials"] if row["eligible"]), key=lambda row: row["images_per_second"], reverse=True)[:3] + finalists = list(dict.fromkeys(row["workers"] for row in top)) + for _ in range(args.confirmation_rounds): + rng.shuffle(finalists) + for concurrency in finalists: + if not measure(concurrency, "confirm", work): + break + except (Exception, KeyboardInterrupt) as error: + report["error"] = f"{type(error).__name__}: {error}" + error_path = args.output.with_suffix(".error.log") + error_path.write_text(traceback.format_exc()) + raise SystemExit(f"Calibration stopped: {error}. Details saved to {error_path}") from None + finally: + report["recommendation"] = recommend(report["trials"], args.tolerance) + report["used_paths"] = cursor + report["notes"] = [ + "Best tested thread concurrency, not a global optimum or a DataLoader process-worker setting.", + "Decode measures CPU image open/RGB conversion and optional resize; no transforms, batching, GPU or DDP.", + "Stage measures buffered writes and close, not fsync durability; read mode discards bytes.", + "Process/thread initialization is excluded from I/O timing and recorded separately. Partial completed reads are eligible.", + "Disjoint paths do not guarantee cold data, balanced file sizes, or independence from shared-service load.", + ] + write_json(args.output, report) + summary_path = args.output.with_suffix(".summary.txt") + summary_path.write_text(summary(report)) + print(f"Compact summary: {summary_path}", flush=True) + if report["recommendation"]["workers"] is None: + print("No completed reads to rank. Increase --trial-seconds or inspect the recorded errors.", flush=True) + print(json.dumps(report["recommendation"], indent=2), flush=True) + print(f"Full timings, errors and disjoint sample manifest: {args.output}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/dev/ucloud/combined-int8.json b/dev/ucloud/combined-int8.json new file mode 100644 index 00000000..34ea1f65 --- /dev/null +++ b/dev/ucloud/combined-int8.json @@ -0,0 +1,40 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-combined-int8-1", + "environments": { + "master": { + "python": "/work/venvs/mt-master/bin/python", + "commit": "2ebbe39f2365bfea482d7ebc3d000cf4d2044918" + }, + "quant": { + "python": "/work/venvs/mt-quant/bin/python", + "commit": "4ea6b9613fc29a8e33ef72f28885f4e2a0e5906b" + } + }, + "gpus": 1, + "global_batch_size": 32, + "num_workers_per_rank": 2, + "epochs": 4, + "size": 384, + "seeds": [ + 42 + ], + "variants": [ + "quant_float_combined", + "master_eager", + "quant_compile_model", + "quant_eager", + "quant_compile_both", + "quant_int8_combined" + ], + "qualification": { + "seed": 42, + "train": 4096, + "validation": 1024, + "test": 128 + }, + "timeout_seconds": 300, + "budget_seconds": 1800, + "require_finite_losses": true, + "figures": false +} diff --git a/dev/ucloud/compare.py b/dev/ucloud/compare.py new file mode 100644 index 00000000..258bfc6a --- /dev/null +++ b/dev/ucloud/compare.py @@ -0,0 +1,496 @@ +"""Single-node UCloud comparison controller; no package imports or installation.""" + +import argparse +import contextlib +import csv +import hashlib +import json +import math +import os +import random +import shutil +import signal +import subprocess +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +VARIANTS = { + "master_eager": ("master", {}), + "quant_eager": ("quant", {}), + "quant_prefetch": ("quant", {"cuda_prefetch": True}), + "quant_compile_model": ("quant", {"compile": True}), + "quant_compile_optimizer": ("quant", {"compile_optimizer": True}), + "quant_compile_both": ("quant", {"compile": True, "compile_optimizer": True}), + "quant_float_combined": ("quant", {"compile": True, "compile_optimizer": True, "cuda_prefetch": True}), + "quant_int8_combined": ("quant", {"quantized_training": True, "compile": True, "compile_optimizer": True, "cuda_prefetch": True}), + "quant_combined": ("quant", {"compile": True, "compile_optimizer": True, "cuda_prefetch": True}), + # Qualification experiments, deliberately excluded from the default matrix. + "quant_model_graphs": ("quant", {"compile": True, "compile_mode": "reduce-overhead"}), + "quant_optimizer_graphs": ("quant", {"compile_optimizer": True, "optimizer_cudagraphs": True}), + "quant_int8": ("quant", {"quantized_training": True}), +} + + +def uses_quantized_training(config): + return any(VARIANTS[name][1].get("quantized_training", False) for name in config["variants"]) + + +def digest(path): + with Path(path).open("rb") as handle: + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def write_json(path, value): + path = Path(path) + temporary = path.with_suffix(".tmp") + with temporary.open("w") as handle: + json.dump(value, handle, indent=2) + handle.write("\n") + temporary.replace(path) + + +def branches(config): + return sorted({VARIANTS[name][0] for name in config["variants"]}) if config.get("mode") == "scaling" else ["master", "quant"] + + +def validate(config): + if config.get("mode", "comparison") not in ("comparison", "scaling"): + raise ValueError("mode must be comparison or scaling") + for key in ("prepare_timeout_seconds", "minimum_gpu_memory_gib"): + if key in config and (type(config[key]) is not int or config[key] < 1): + raise ValueError(f"{key} must be a positive integer") + for key in ("full_taxonomy", "wandb"): + if key in config and type(config[key]) is not bool: + raise ValueError(f"{key} must be a boolean") + if config.get("checkpoint"): + if type(config.get("resume_epoch")) is not int or not 0 < config["resume_epoch"] < config["epochs"]: + raise ValueError("checkpoint requires resume_epoch between zero and epochs") + config["checkpoint"] = str(Path(config["checkpoint"]).expanduser().resolve()) + elif "resume_epoch" in config: + raise ValueError("resume_epoch requires checkpoint") + if config.get("reuse_preparation"): + config["reuse_preparation"] = str(Path(config["reuse_preparation"]).expanduser().resolve()) + if "figures" in config and type(config["figures"]) is not bool: + raise ValueError("figures must be a boolean") + if "require_finite_losses" in config and type(config["require_finite_losses"]) is not bool: + raise ValueError("require_finite_losses must be a boolean") + if "budget_seconds" in config and (type(config["budget_seconds"]) is not int or config["budget_seconds"] < 1): + raise ValueError("budget_seconds must be a positive integer") + if config.get("exclude_qualification"): + if config.get("reuse_preparation") or not config.get("qualification"): + raise ValueError("Disjoint selection requires fresh qualification preparation") + config["exclude_qualification"] = str(Path(config["exclude_qualification"]).resolve()) + expected = config.get("exclude_qualification_sha256", "") + if len(expected) != 64 or any(c not in "0123456789abcdef" for c in expected): + raise ValueError("Disjoint selection requires a pinned exclusion hash") + qualification = config.get("qualification") + if qualification is not None: + if not isinstance(qualification, dict) or set(qualification) != {"seed", "train", "validation", "test"}: + raise ValueError("qualification requires exactly seed, train, validation and test") + if type(qualification["seed"]) is not int: + raise ValueError("qualification.seed must be an integer") + if any(type(qualification[key]) is not int or qualification[key] < 1 for key in ("train", "validation", "test")): + raise ValueError("qualification split sizes must be positive integers") + for key in ("gpus", "global_batch_size", "epochs", "size", "timeout_seconds"): + if type(config[key]) is not int or config[key] < 1: + raise ValueError(f"{key} must be a positive integer") + if config["gpus"] not in (1, 2, 4, 8): + raise ValueError("Choose 1, 2, 4 or 8 GPUs on one node") + if config["global_batch_size"] % config["gpus"]: + raise ValueError("global_batch_size must divide evenly across GPUs") + if qualification is not None and qualification["train"] < config["global_batch_size"]: + raise ValueError("qualification.train must contain at least one global batch") + if type(config["num_workers_per_rank"]) is not int or config["num_workers_per_rank"] < 0: + raise ValueError("num_workers_per_rank must be a nonnegative integer") + if not config["seeds"] or any(type(seed) is not int for seed in config["seeds"]): + raise ValueError("Supply integer seeds") + if len(set(config["seeds"])) != len(config["seeds"]): + raise ValueError("Duplicate seeds") + if not config["variants"] or len(set(config["variants"])) != len(config["variants"]): + raise ValueError("Supply unique variants") + if set(config["variants"]) - VARIANTS.keys(): + raise ValueError("Unknown variant") + if uses_quantized_training(config) and config["gpus"] != 1: + raise ValueError("Native INT8 training requires gpus=1; DDP is unsupported") + if config.get("mode") != "scaling" and not {"master_eager", "quant_eager"}.issubset(config["variants"]): + raise ValueError("Keep both eager controls") + for branch in branches(config): + entry = config["environments"][branch] + if len(entry["commit"]) != 40 or any(c not in "0123456789abcdef" for c in entry["commit"]): + raise ValueError("Pin each environment to a full Git commit") + entry["python"] = str(Path(entry["python"]).expanduser().absolute()) + for key in ("parquet", "output"): + config[key] = str(Path(config[key]).expanduser().resolve()) + return config + + +def plan(config): + runs = [] + for seed in config["seeds"]: + variants = config["variants"].copy() + random.Random(seed).shuffle(variants) + for variant in variants: + branch, options = VARIANTS[variant] + runs.append({"name": f"{variant}_seed{seed}", "seed": seed, "branch": branch, "options": options}) + return runs + + +def reuse_preparation(config, output): + """Copy verified model/data artifacts; keep the new trial's budget and plan.""" + source = Path(config["reuse_preparation"]) + original = json.loads((source / "comparison.json").read_text()) + for key in ( + "parquet", + "size", + "seeds", + "qualification", + "full_taxonomy", + "environments", + "exclude_qualification", + "exclude_qualification_sha256", + ): + if config.get(key) != original.get(key): + raise ValueError(f"Cannot reuse preparation with different {key}") + manifest = json.loads((source / "prepared.json").read_text()) + for name, expected in manifest.items(): + if digest(source / name) != expected: + raise ValueError(f"Prepared artifact changed: {name}") + if name.endswith(".py") and digest(HERE / name) != expected: + raise ValueError(f"Harness changed after preparation: {name}") + if digest(config["parquet"]) != json.loads((source / "dataset.json").read_text())["parquet_sha256"]: + raise ValueError("Source Parquet changed") + for name in manifest: + if not name.endswith(".py") and name != "budget.json": + shutil.copyfile(source / name, output / name) + (output / "prepare.log").write_text(f"Reused verified preparation from {source}\n") + + +def command(config, run, config_path): + python = config["environments"][run["branch"]]["python"] + worker = [str(HERE / "worker.py"), "train", str(config_path), "--run", run["name"]] + if config["gpus"] == 1: + return [python, *worker] + return [ + python, + "-m", + "torch.distributed.run", + "--standalone", + "--nnodes=1", + f"--nproc-per-node={config['gpus']}", + "--max-restarts=0", + "--", # Stop torchrun from abbreviating worker flags such as --run. + *worker, + ] + + +def execute(args, log, cwd, timeout): + """Kill the entire torchrun process group on timeout or interruption.""" + env = dict(os.environ) + env.pop("PYTHONPATH", None) + env["PYTHONNOUSERSITE"] = "1" + env["PYTHONUNBUFFERED"] = "1" + for key in ("RANK", "WORLD_SIZE", "LOCAL_RANK", "MASTER_ADDR", "MASTER_PORT"): + env.pop(key, None) + with Path(log).open("w") as handle: + proc = subprocess.Popen(args, cwd=cwd, env=env, stdout=handle, stderr=subprocess.STDOUT, start_new_session=True) + try: + return proc.wait(timeout=timeout) + except BaseException: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + # The leader may exit before its loader/compiler children do. + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + raise + + +def audit_losses(path, epochs, start_epoch=0): + """Check recorded train/eval losses, without changing the measured GPU loop.""" + expected = {(str(epoch), phase) for epoch in range(start_epoch, epochs) for phase in ("train", "eval")} + seen = set() + issues = [] + try: + with Path(path).open() as handle: + for row in csv.DictReader(handle): + key = (row.get("epoch"), row.get("type")) + if key not in expected or key in seen: + issues.append(f"Unexpected or duplicate epoch/phase: {key}") + seen.add(key) + for column in ("loss", "loss/lvl0", "loss/lvl1", "loss/lvl2"): + try: + finite = math.isfinite(float(row.get(column, ""))) + except (TypeError, ValueError): + finite = False + if not finite: + issues.append(f"epoch={key[0]} phase={key[1]} {column}={row.get(column)!r}") + except OSError as error: + issues.append(str(error)) + issues.extend(f"Missing epoch={epoch} phase={phase}" for epoch, phase in sorted(expected - seen)) + return issues + + +def summary(output): + rows = [json.loads(p.read_text()) for p in sorted((output / "runs").glob("*/result.json"))] + for row in rows: + phases = [ + [json.loads(line) for line in path.read_text().splitlines()] + for path in sorted((output / "runs" / row["name"]).glob("phases-rank*.jsonl")) + ] + if phases: + training = [[p["seconds"] for p in rank if p["phase"] == "train"] for rank in phases] + if training and all(len(t) == len(training[0]) for t in training) and training[0]: + times = [max(epoch) for epoch in zip(*training, strict=True)] + row["train_seconds"] = sum(times) + row["first_epoch_seconds"] = times[0] + row["later_epoch_mean_seconds"] = sum(times[1:]) / len(times[1:]) if len(times) > 1 else "" + row["peak_allocated_bytes"] = max(p["max_memory_allocated"] for rank in phases for p in rank) + scores = [p["selection_metric"] for p in phases[0] if p["phase"] == "validation"] + row["best_validation_metric"] = max(scores) if scores else "" + fields = [ + "name", + "branch", + "seed", + "status", + "loss_check", + "scope", + "returncode", + "wall_seconds", + "train_seconds", + "first_epoch_seconds", + "later_epoch_mean_seconds", + "peak_allocated_bytes", + "best_validation_metric", + "checkpoint", + "checkpoint_sha256", + ] + with (output / "comparison.csv").open("w") as handle: + writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + controls = {row["seed"]: row for row in rows if row["name"].startswith("master_eager_") and row["status"] == "completed"} + paired = [] + for row in rows: + if row["status"] == "completed" and row["seed"] in controls: + control = controls[row["seed"]] + paired.append( + { + "name": row["name"], + "seed": row["seed"], + "scope": row.get("scope", "full_dataset"), + "wall_speedup_vs_master": control["wall_seconds"] / row["wall_seconds"], + "validation_delta_vs_master": ( + row["best_validation_metric"] - control["best_validation_metric"] + if isinstance(row.get("best_validation_metric"), (int, float)) + and isinstance(control.get("best_validation_metric"), (int, float)) + else None + ), + } + ) + write_json(output / "paired.json", paired) + + +class BudgetExceeded(TimeoutError): + pass + + +@contextlib.contextmanager +def execution_budget(config, stage): + """One wall-clock deadline across prepare and train, including time between commands.""" + if "budget_seconds" not in config or stage not in ("prepare", "train"): + yield None + return + if stage == "prepare": + deadline = time.time() + config["budget_seconds"] + if config.get("mode") == "scaling" and config.get("reuse_preparation"): + source_budget = Path(config["reuse_preparation"]) / "budget.json" + deadline = min(deadline, json.loads(source_budget.read_text())["deadline"]) + else: + path = Path(config["output"]) / "budget.json" + if not path.is_file(): + raise ValueError("Preparation budget record is missing; complete preparation in a new output directory") + deadline = json.loads(path.read_text())["deadline"] + remaining = deadline - time.time() + if remaining <= 0: + raise BudgetExceeded("Comparison time budget expired; preserve results and use a new output for another attempt") + + def expired(signum, frame): + raise BudgetExceeded("Comparison time budget expired; active workers are being stopped") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, remaining) + try: + yield deadline + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def main(): + def terminate(signum, frame): + raise KeyboardInterrupt(f"Received signal {signum}") + + signal.signal(signal.SIGTERM, terminate) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config", type=Path) + parser.add_argument("--stage", choices=("plan", "prepare", "train", "summary"), default="plan") + parser.add_argument("--only", help="Run one planned name (for a qualification run)") + args = parser.parse_args() + config = validate(json.loads(args.config.read_text())) + try: + with execution_budget(config, args.stage) as deadline: + run_stage(config, args, deadline) + except BudgetExceeded as error: + raise SystemExit(str(error)) from None + except KeyboardInterrupt: + raise SystemExit(130) from None + + +def run_stage(config, args, deadline=None): + runs = plan(config) + output = Path(config["output"]) + config_path = output / "comparison.json" + if args.stage == "plan": + print( + json.dumps( + { + "per_rank_batch": config["global_batch_size"] // config["gpus"], + "scope": "qualification_subset" if config.get("qualification") else "full_dataset", + "qualification": config.get("qualification"), + "budget_seconds": config.get("budget_seconds"), + "runs": runs, + }, + indent=2, + ) + ) + return + if args.stage == "summary": + summary(output) + return + if args.stage == "prepare": + if output.exists(): + raise ValueError(f"Output already exists: {output}. Preserve it and choose a new output path; preparation cannot resume") + output.mkdir(parents=True, exist_ok=False) + (output / "runs").mkdir() + if deadline is not None: + write_json(output / "budget.json", {"deadline": deadline}) + write_json(config_path, config) + write_json(output / "plan.json", runs) + # Preserve the harness itself independently of either installed branch. + for file in HERE.glob("*.py"): + (output / file.name).write_bytes(file.read_bytes()) + for branch in branches(config): + print(f"Checking {branch}; log: {output}/preflight-{branch}.log", flush=True) + result = execute( + [config["environments"][branch]["python"], str(HERE / "worker.py"), "preflight", str(config_path), "--branch", branch], + output / f"preflight-{branch}.log", + output, + 300, + ) + if result: + raise RuntimeError(f"Preflight failed: see {output}/preflight-{branch}.log") + if set(branches(config)) == {"master", "quant"}: + master = json.loads((output / "environment-master.json").read_text()) + quant = json.loads((output / "environment-quant.json").read_text()) + if master["dependencies"] != quant["dependencies"] or master["python_version"] != quant["python_version"]: + raise RuntimeError("Environment versions differ; use the same locked dependencies and Python for both branches") + print(f"Preparing dataset and starting weights; follow: tail -f {output}/prepare.log", flush=True) + if config.get("reuse_preparation"): + reuse_preparation(config, output) + result = 0 + else: + preparation_branch = "master" if "master" in branches(config) else "quant" + result = execute( + [config["environments"][preparation_branch]["python"], str(HERE / "worker.py"), "prepare", str(config_path)], + output / "prepare.log", + output, + config.get("prepare_timeout_seconds", config["timeout_seconds"]), + ) + if result: + raise RuntimeError(f"Preparation failed: see {output}/prepare.log; use a new output for a fresh attempt") + files = ["class_spec.json", "data_index.json", "dataset.json", "preprocessing.txt"] + files += [p.name for p in HERE.glob("*.py")] + files += [f"initial_seed{s}.pt" for s in config["seeds"]] + if config.get("qualification"): + files.extend(["qualification.parquet", "selection.json"]) + if deadline is not None: + files.append("budget.json") + if config.get("checkpoint"): + write_json(output / "resume-source.json", {"path": config["checkpoint"], "sha256": digest(config["checkpoint"])}) + files.append("resume-source.json") + write_json(output / "prepared.json", {name: digest(output / name) for name in files}) + print(f"Preparation complete: {output}/prepared.json", flush=True) + return + if not (output / "prepared.json").is_file(): + raise ValueError(f"Preparation is incomplete: {output}. Check the active process and prepare.log before starting a new attempt") + if json.loads(config_path.read_text()) != config: + raise ValueError("Configuration differs from the frozen preparation; use a new output directory") + for name, sha in json.loads((output / "prepared.json").read_text()).items(): + if digest(output / name) != sha: + raise ValueError(f"Prepared artifact changed: {name}") + if name.endswith(".py") and digest(HERE / name) != sha: + raise ValueError(f"Harness changed after preparation: {name}") + if digest(config["parquet"]) != json.loads((output / "dataset.json").read_text())["parquet_sha256"]: + raise ValueError("Source Parquet changed") + if config.get("checkpoint"): + source = json.loads((output / "resume-source.json").read_text()) + if digest(config["checkpoint"]) != source["sha256"]: + raise ValueError("Resume checkpoint changed since preparation") + if args.only and args.only not in {run["name"] for run in runs}: + raise ValueError("--only must match a name in --stage plan") + failed = False + for run in runs: + if args.only and args.only != run["name"]: + continue + directory = output / "runs" / run["name"] + if directory.exists(): + record = directory / "result.json" + if record.exists(): + result = json.loads(record.read_text()) + if result["status"] == "completed" and digest(result["checkpoint"]) == result["checkpoint_sha256"]: + continue + raise RuntimeError(f"Incomplete/failed run exists: {directory}. Preserve it and use a new comparison directory") + directory.mkdir() + args_run = command(config, run, config_path) + write_json(directory / "launch.json", {**run, "argv": args_run}) + result = {**run, "status": "running", "scope": "qualification_subset" if config.get("qualification") else "full_dataset"} + write_json(directory / "result.json", result) + print(f"Starting {run['name']}; log: {directory}/console.log", flush=True) + started = time.monotonic() + try: + result["returncode"] = execute(args_run, directory / "console.log", output, config["timeout_seconds"]) + checkpoint = directory / "model" / "weights" / "best.pt" + result["status"] = "completed" if result["returncode"] == 0 and checkpoint.is_file() else "failed" + if result["status"] == "completed": + result.update(checkpoint=str(checkpoint), checkpoint_sha256=digest(checkpoint)) + if config.get("require_finite_losses", False): + issues = audit_losses(directory / "model" / "logs" / "summary.csv", config["epochs"], config.get("resume_epoch", 0)) + result["loss_check"] = "failed" if issues else "passed" + if issues: + result.update(status="invalid_metrics", loss_issues=issues) + except KeyboardInterrupt: + result.update(status="interrupted", error="Interrupted by user or termination signal") + raise SystemExit(130) from None + except (BudgetExceeded, subprocess.TimeoutExpired) as error: + result.update(status="timed_out", error=str(error)) + raise + except BaseException as error: + result.update(status="failed", error=repr(error)) + raise + finally: + result["wall_seconds"] = time.monotonic() - started + write_json(directory / "result.json", result) + summary(output) + failed |= result["status"] != "completed" + if failed: + raise SystemExit("Some runs failed; see comparison.csv and individual console logs") + + +if __name__ == "__main__": + main() diff --git a/dev/ucloud/comparison.json b/dev/ucloud/comparison.json new file mode 100644 index 00000000..38cbe0a6 --- /dev/null +++ b/dev/ucloud/comparison.json @@ -0,0 +1,16 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-ddp", + "environments": { + "master": {"python": "/work/venvs/mt-master/bin/python", "commit": "2ebbe39f2365bfea482d7ebc3d000cf4d2044918"}, + "quant": {"python": "/work/venvs/mt-quant/bin/python", "commit": "4ea6b9613fc29a8e33ef72f28885f4e2a0e5906b"} + }, + "gpus": 4, + "global_batch_size": 64, + "num_workers_per_rank": 8, + "epochs": 10, + "size": 384, + "seeds": [42, 43, 44], + "variants": ["master_eager", "quant_eager", "quant_prefetch", "quant_compile_model", "quant_compile_optimizer", "quant_combined"], + "timeout_seconds": 115200 +} diff --git a/dev/ucloud/compilation.json b/dev/ucloud/compilation.json new file mode 100644 index 00000000..3d05fd71 --- /dev/null +++ b/dev/ucloud/compilation.json @@ -0,0 +1,37 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-compile-model-1", + "environments": { + "master": { + "python": "/work/venvs/mt-master/bin/python", + "commit": "2ebbe39f2365bfea482d7ebc3d000cf4d2044918" + }, + "quant": { + "python": "/work/venvs/mt-quant/bin/python", + "commit": "4ea6b9613fc29a8e33ef72f28885f4e2a0e5906b" + } + }, + "gpus": 1, + "global_batch_size": 32, + "num_workers_per_rank": 2, + "epochs": 4, + "size": 384, + "seeds": [ + 42 + ], + "variants": [ + "master_eager", + "quant_eager", + "quant_compile_model" + ], + "qualification": { + "seed": 42, + "train": 4096, + "validation": 1024, + "test": 128 + }, + "timeout_seconds": 300, + "budget_seconds": 1800, + "require_finite_losses": true, + "figures": false +} diff --git a/dev/ucloud/ddp.json b/dev/ucloud/ddp.json new file mode 100644 index 00000000..b70021c6 --- /dev/null +++ b/dev/ucloud/ddp.json @@ -0,0 +1,36 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-ddp-4gpu-b32-1", + "environments": { + "quant": { + "python": "/work/venvs/mt-quant/bin/python", + "commit": "4a1bd8dda55d668754a2bd628060039e5cc4bd10" + } + }, + "gpus": 4, + "global_batch_size": 128, + "num_workers_per_rank": 4, + "epochs": 3, + "size": 384, + "seeds": [ + 42 + ], + "variants": [ + "quant_compile_model" + ], + "qualification": { + "seed": 42, + "train": 32768, + "validation": 4096, + "test": 128 + }, + "timeout_seconds": 600, + "budget_seconds": 1800, + "require_finite_losses": true, + "figures": true, + "mode": "scaling", + "minimum_gpu_memory_gib": 140, + "full_taxonomy": true, + "wandb": true, + "prepare_timeout_seconds": 900 +} diff --git a/dev/ucloud/ddp.md b/dev/ucloud/ddp.md new file mode 100644 index 00000000..0eda954b --- /dev/null +++ b/dev/ucloud/ddp.md @@ -0,0 +1,334 @@ +# Four-GPU qualification and production handoff + +Use one manually allocated UCloud node with **four full B200 GPUs**, the actual +Parquet and image storage, and internet access. `ddp.json` rejects devices below +140 GiB each, so this is not a continuation on the fractional-GPU job. + +The chosen configuration is floating-point training on `quant`: FP16 AMP, model +compilation, figures, W&B, loss auditing and checkpoints. Optimizer compilation, +explicit CUDA prefetch, INT8 and EMA remain off. Qualification alone uses the +Python API. Production will use `mt_htrain` under `torchrun`. + +This campaign qualifies four ranks. It does not establish eight-rank scaling or +full-dataset I/O throughput. If eight GPUs become available later, qualify that +allocation separately. At the same per-GPU batch, four GPUs halve the global batch; +review the production batch and learning-rate schedule explicitly before training. + +## Setup and authentication + +Clone the repository on branch `quant` into `/work/mini_trainer` (or pull it there). +Use the same checkout throughout the campaign; preparation hashes the harness. +With `uv`, `git`, Python and a C++ compiler available: + +```bash +cd /work/mini_trainer +git pull --ff-only +MT_TEMPLATE=dev/ucloud/ddp.json MT_CONFIG=/work/ddp-b32.json \ + bash dev/ucloud/setup.sh \ + /work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet + +# Authenticate in the foreground before redirected workers start. +/work/venvs/mt-quant/bin/wandb login +# Optional: WANDB_ENTITY selects a team; otherwise use the authenticated default. +export WANDB_PROJECT=mini-trainer-ddp +nvidia-smi -L +``` + +Authentication uses the existing W&B SDK, without a new login mechanism or +credentials in Git. Each trial uses one shared W&B run with rank labels and a +unique ID. Only rank zero uploads figures and controls the shared finish state. +See [W&B distributed logging](https://docs.wandb.ai/models/track/log/distributed-training). +The installed quant environment contains the pinned package and locked dependencies. + +## Baseline and batch sweep + +Preparation samples 32,768 training, 4,096 validation and 128 test images from the +**existing splits**, but builds the head from the **full source taxonomy**. It +preserves class order across trials. The subset is half the eight-GPU proposal, +keeping 256 training steps and 32 validation steps per rank at batch 32. +This tests production head/figure dimensions +without training on all six million rows. Read `prepare.log` for actual class +counts. The test split is reserved for later evaluation, not batch selection. + +There is one **30-minute wall-clock budget** shared by baseline preparation and +all derived trials, including gaps between commands. Individual training workers +have a 600-second guard; initial preparation has a 900-second guard. Neither is +an expected duration. If the campaign expires, retain its results and review +before starting another qualification campaign in the same allocated job. +Figures and W&B stay enabled. Qualification deadlines stop child processes, not +the UCloud allocation. + +Define this helper in the foreground terminal or tmux session: + +```bash +cd /work/mini_trainer +run_trial() { + local cfg="$1" root rc=0 + root=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["output"])' "$cfg") || return + bash dev/ucloud/launch.sh "$cfg" --stage plan && + bash dev/ucloud/launch.sh "$cfg" --stage prepare && + bash dev/ucloud/launch.sh "$cfg" --stage train || rc=$? + if [[ -f "$root/prepared.json" ]]; then + bash dev/ucloud/launch.sh "$cfg" --stage summary || rc=$? + python3 dev/ucloud/scaling.py report "$root" || rc=$? + fi + return "$rc" +} +run_trial /work/ddp-b32.json +``` + +The baseline is three epochs at 32 images per GPU (global batch 128), with four +loader workers per rank. Four GPUs are launched with `torchrun`, no Slurm. +Check all four ranks completed, losses passed, figures look correct in W&B, +and reserved GPU memory has headroom before increasing the batch: + +```bash +python3 dev/ucloud/scaling.py trial /work/ddp-b32.json /work/ddp-b64.json \ + --output /work/results/global-lepi-ddp-4gpu-b64-1 --batch 64 +run_trial /work/ddp-b64.json + +# Only after the 64/GPU result passes and has memory headroom: +python3 dev/ucloud/scaling.py trial /work/ddp-b32.json /work/ddp-b128.json \ + --output /work/results/global-lepi-ddp-4gpu-b128-1 --batch 128 +run_trial /work/ddp-b128.json +``` + +Derived trials reuse verified preparation and starting weights, and inherit the +original deadline. They do not scan/sample/build the taxonomy again. Do not edit +frozen configs or reuse a failed output directory. Stop increasing at an OOM, +failed loss check, unstable memory, or lack of throughput improvement. Consider +256/GPU only if 128/GPU still has substantial headroom and time remains. + +Select by **warm aggregate training images/second at an acceptable memory +footprint**, reserving roughly 20% GPU memory, rather than speed at fixed batch. +`scaling.py report` uses summed sample counts divided by the slowest rank's phase +time and excludes the first epoch, which includes cold compilation. It reports +peak reserved memory as a fraction of actual device capacity. W&B also records +`qualification/*` phase metrics; rank-level evidence remains in `phases-rank*.jsonl`. + +Loader wait measures host time obtaining batches, including worker startup, +loading and decoding; it is not a pure storage measurement and can overlap GPU +work. Compare it alongside throughput. If it is high across warm epochs, try +one matched trial with more loader workers instead of a broad matrix. The +repeated subset benefits from filesystem caches: these timings cannot certify +full-dataset distributed-I/O throughput for a 24-hour run. + +## Stability and checkpoint continuation + +Reserve time for this gate instead of chasing every batch size. Set `BATCH` to +the selected **per-GPU** batch. Request a four-epoch scheduler horizon and preserve the generated horizon for +both the source and restored trial: + +```bash +BATCH=64 # replace with the measured choice +WORKERS=4 # replace with the measured worker count +python3 dev/ucloud/scaling.py trial /work/ddp-b32.json /work/ddp-stability.json \ + --output /work/results/global-lepi-ddp-4gpu-stability-1 --batch "$BATCH" --workers "$WORKERS" --epochs 4 +run_trial /work/ddp-stability.json +HORIZON=$(python3 -c 'import json; print(json.load(open("/work/ddp-stability.json"))["epochs"])') + +python3 dev/ucloud/scaling.py trial /work/ddp-b32.json /work/ddp-restore.json \ + --output /work/results/global-lepi-ddp-4gpu-restore-1 --batch "$BATCH" --workers "$WORKERS" --epochs "$HORIZON" \ + --checkpoint /work/results/global-lepi-ddp-4gpu-stability-1/runs/quant_compile_model_seed42/model/weights/checkpoint_1.pth \ + --resume-epoch 2 +run_trial /work/ddp-restore.json +``` + +The restored run starts at epoch 2 (zero-based), trains the remaining epochs, +and uses a new output and W&B run. Each rank checks model, optimizer, scheduler +and scaler state against the source checkpoint **before any updates**. Require +four successful `restore-rank*.json` records, finite train/eval losses, stable +memory, and working figures/checkpoints after restoration. This tests state +restoration and continued execution, not bitwise reproduction of augmentation RNG. + +## Keep the allocation and hand off immediately + +Qualification and production run in the **same allocated UCloud job**. Do not +terminate/release the job, rebuild its environment, or wait for a new allocation +between them. The separate storage campaign below means a separate harness budget +and output directory, not another UCloud job. Do not enable the continuous-benchmark +provisioning/cleanup policy for this production allocation. + +Before qualification, prepare the production CLI configuration and verify its paths, +full-dataset splits, pinned environment, W&B authentication, output location and +learning-rate/epoch recipe. Leave only measured batch and worker choices to finalize. +Keep the selected environment and caches warm. Reserve qualification time **in +addition to** the intended production duration in the UCloud job lifetime, plus +checkpoint/finalization margin; verify remaining time before launch. If the job +lifetime is insufficient, resolve its extension or a shorter reviewed training +budget while qualification runs rather than releasing the allocation. + +After the required gates pass, record the selected batch/workers and qualification +report, finalize the pre-reviewed production configuration, and launch `mt_htrain` +under four-rank `torchrun` immediately in the existing terminal/tmux session. Do not +spend the allocation completing optional sweep points once the choice is supported. +The production configuration is still an explicit prerequisite, not generated or +validated by these qualification commands. + +## Production uses the public CLIs + +Do not launch the 24-hour run from this qualification harness. After qualification, +freeze a reviewed `production.yaml` for the full dataset, selected batch and +worker count, intended learning-rate schedule, complete taxonomy and preprocessing. +Record package/dependency pins, W&B run, split provenance and checkpoint hashes. +The final epoch count and learning-rate recipe need a separate production decision; +short qualification accuracy does not select them. + +The planned command interface is: + +```bash +/work/venvs/mt-quant/bin/python -m torch.distributed.run \ + --standalone --nnodes=1 --nproc-per-node=4 --max-restarts=0 --no-python \ + /work/venvs/mt-quant/bin/mt_htrain --config /work/production.yaml --wandb --compile + +/work/venvs/mt-quant/bin/mt_hpredict --config /work/evaluation.yaml +uvx --from "mini_metrics @ git+https://github.com/asgersvenning/mini_metrics.git@$METRICS_SHA" \ + mm_metrics --files "$PREDICTIONS_CSV" --output-dir /work/production-metrics --output + +/work/venvs/mt-quant/bin/mt_export --weights "$BEST_PT" \ + --output /work/production-onnx --input-shape 3 384 384 \ + --preprocessing /work/deployment-preprocessing.json +``` + +These production/evaluation files and shell variables are **handoff placeholders**, +not runnable configs yet. `METRICS_SHA` must be a reviewed immutable commit. +Evaluation must use only the original held-out test split, with matching class +order and score semantics. Use the existing `mt_hpredict` CLI for this hierarchical +model; it also provides an explicit flat-head route. Consolidating inference into +one CLI is [deferred on the roadmap](../../docs/roadmap.md#5-mini_metrics-and-continuous-model-evaluation), +not a production prerequisite. Verify the deployment preprocessing and real-image +ONNX parity following [the export guide](../../docs/onnx.md). + +A later full `master` training comparison remains optional and separate. This +qualification deliberately selects and stress-tests one configuration. + +## Bounded resource-utilization decisions + +Run only the next informative trial; do not execute this as an unattended matrix. +Keep the initial 30-minute campaign and reserve time for checkpoint continuation. +A minimum campaign is baseline, one larger batch, one worker comparison, then +restoration. The time guards can expire before all gates finish; retain evidence +and review any additional campaign rather than silently extending its budget. + +Trial generation automatically raises the epoch count when needed to plan at least +100 training steps after the first epoch. At 256/GPU this subset requires five +rather than three epochs. `scaling.py report` exposes actual `warm_steps` and +`sufficient_warm_steps`; incomplete/failed runs are not performance-qualified merely +because their planned duration was sufficient. Changing the epoch horizon also +changes the learning-rate schedule, so these are throughput trials, not matched +convergence comparisons. + +After selecting a batch, compare workers while holding batch, subset and epoch +horizon fixed. Start with 8/GPU; try 16/GPU only if loading/throughput evidence +justifies it. A small timing difference is not enough to select a winner. + +```bash +BATCH=64 # replace with measured choice +python3 dev/ucloud/scaling.py trial /work/ddp-b32.json /work/ddp-w8.json \ + --output /work/results/global-lepi-ddp-4gpu-w8-1 --batch "$BATCH" --workers 8 +run_trial /work/ddp-w8.json + +# Optional, after reviewing w8: +python3 dev/ucloud/scaling.py trial /work/ddp-b32.json /work/ddp-w16.json \ + --output /work/results/global-lepi-ddp-4gpu-w16-1 --batch "$BATCH" --workers 16 +run_trial /work/ddp-w16.json +``` + +Use `--workers "$WORKERS"` for both stability and restoration trials once workers +are selected. Read the generated stability config's actual `epochs` and pass that +same value to the restore trial; automatic warm-step sizing may have increased it. +A restore run keeps its explicit horizon and excludes its own first epoch from warm +performance reporting because it starts a fresh compiled process. + +Reports retain per-rank phase records, training rank-time ratios, allocation peaks +and 32-batch timing windows (including a final partial window). Completed windows are also appended immediately +to `windows-rank*.jsonl` so a timeout retains them. Windows synchronize +CUDA at their boundaries, adding some overhead consistently across trials. They +measure rank-local elapsed time, not pure disk throughput. Figure and checkpoint +call durations are recorded separately in `components-rank*.jsonl`; figures are +also inside validation phase time, so do not add these overlapping totals. These +component measurements are host elapsed time and include backend logging costs. + +### Separate broader storage pass + +This is explicitly launched with its own budget. It requires fresh preparation, +selects 262,144 training / 8,192 validation / 128 test images, and excludes all image +identities in the baseline selection. The full taxonomy remains in use. The +baseline selection hash is checked, and `selection.json` records the new Parquet +hash and exclusion provenance; both are frozen in the preparation manifest. + +```bash +BATCH=64 # selected batch +WORKERS=8 # selected worker count +python3 dev/ucloud/scaling.py trial /work/ddp-b32.json /work/ddp-storage.json \ + --output /work/results/global-lepi-ddp-4gpu-storage-1 \ + --batch "$BATCH" --workers "$WORKERS" --storage +run_trial /work/ddp-storage.json +``` + +The worker runs one epoch with a 600-second guard; the separate campaign allows +1,800 seconds including preparation (up to 900 seconds) and operator gaps. These +are limits, not runtime estimates or UCloud allocation extensions. A timeout may +leave partial window/phase evidence and must not be reported as success. With only +one epoch there is no warm-epoch result: inspect successive windows and first-epoch +cost separately. Reads use actual dataset storage without explicit RAM caching; +do not flush shared caches. Disjoint images reduce reuse of the baseline sample +but do not establish cold-cache conditions or certify full-dataset production I/O. + +## Generate the production CLI configuration + +`production.py` verifies the starting weights and full-taxonomy preparation artifacts +and writes a new YAML file for the real `mt_htrain` CLI. It deliberately omits the +qualification `data_index`: the CLI parses the complete source Parquet and preserves +its supplied splits. The starting weights are the original pretrained initialization, +not a model trained on the qualification subset. Full metadata parsing/broadcast and +full validation have larger CPU/memory costs than qualification. + +Generate this only after the epoch horizon is reviewed from the broader storage +measurement and remaining allocation time. The generator requires an explicit epoch +count; it does not extrapolate the cached subset into a production schedule. It +retains the qualified LR 0.001, weight decay 0.01 and 0.25-epoch warmup; larger global +batches are not automatically assigned a linearly scaled learning rate. + +```bash +# Set these to the final measured/reviewed choices before running this block. +: "${BATCH:?selected per-GPU batch}" +: "${WORKERS:?selected per-GPU workers}" +: "${EPOCHS:?reviewed full-data epoch horizon}" +PRODUCTION_OUTPUT=/work/results # user-confirmed persistent storage for this job + +/work/venvs/mt-quant/bin/python dev/ucloud/production.py \ + /work/results/global-lepi-ddp-4gpu-b32-2 /work/production.yaml \ + --output "$PRODUCTION_OUTPUT" --name global-lepi-production-1 \ + --batch "$BATCH" --workers "$WORKERS" --epochs "$EPOCHS" +``` + +Before launch, verify the generated YAML, available storage and the installed package +pin against the successful qualification. Keep the same four GPUs and environment. +The W&B project and shared run identity are set explicitly in YAML; `--wandb` is still +required to enable the backend. Also pass `--compile` explicitly: the pinned CLI +parser otherwise overrides YAML compilation with its false flag default. +No UCloud credentials are involved. + +```bash +export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 TORCHINDUCTOR_COMPILE_THREADS=1 +export MPLBACKEND=Agg PYTHONHASHSEED=0 TORCH_NCCL_ASYNC_ERROR_HANDLING=1 +export TORCH_HOME=/work/.cache/torch +/work/venvs/mt-quant/bin/python -m torch.distributed.run \ + --standalone --nnodes=1 --nproc-per-node=4 --max-restarts=0 --no-python -- \ + /work/venvs/mt-quant/bin/mt_htrain --config /work/production.yaml --wandb --compile +``` + +Run in tmux and retain stdout/stderr. The pinned CLI writes `checkpoint_last.pth` +after every completed epoch, `best.pt` after improvements, and numbered checkpoints +every five epochs (zero-based 0, 5, 10, ...). These writes are not guaranteed atomic: +if expiry interrupts a write, retain a previous intact numbered checkpoint. The +qualification harness forces more frequent numbered checkpoints; that override does +not carry over to the CLI. Do not assume a final `last.pt` exists after interruption. +The best score on resume is not persisted by the current training loop; archive prior +best artifacts before any separately reviewed resume into an existing output. + +The CLI does not inherit qualification-only finite-loss auditing or timing wrappers. +Inspect the epoch summary and checkpoints during production; the full-dataset run +has no harness wall-time guard and stops at its configured epoch horizon or when the +job/process is terminated. Evaluation/export are separate post-training activities. diff --git a/dev/ucloud/evaluate-results.md b/dev/ucloud/evaluate-results.md new file mode 100644 index 00000000..1df4a561 --- /dev/null +++ b/dev/ucloud/evaluate-results.md @@ -0,0 +1,94 @@ +# Production evaluation with mini_metrics + +Run expert evaluation as soon as expert inference has completed: + +```bash +cd /work/mini_trainer +git pull --ff-only +bash dev/ucloud/evaluate-results.sh expert /work/evaluation-1 +``` + +After staged test inference completes, add its evaluation to the same root: + +```bash +bash dev/ucloud/evaluate-results.sh test /work/evaluation-1 +``` + +Alternatively use `all` once both predictions exist. Each dataset directory must +be fresh; use a new root for reruns. Defaults are: + +- Expert: `/work/expert-full-2/predictions/mini_metric.csv`. +- Test: `/work/test-full-1/predictions/mini_metric.csv`. + +Override these with `MT_EXPERT_CSV` and `MT_TEST_CSV` if inference used another +output directory. Only run against completed inference outputs; file existence +alone is not a completeness guarantee. Confirm the inference exit status and +staging manifest count (expert: 58,640; test: the saved test split count). + +The script uses Python 3.13 through `uvx` and pins mini_metrics to revision +`70cc69adc05362863439277048e06386c1f885e1`. It leaves the training environment +unchanged and uses no GPU. Dependency versions are resolved by uv, not locked +by this helper. Internet access is required for initial tool installation. + +Each dataset gets: + +- `all_labels.csv` and `.json`: aggregate metrics including unseen labels. +- `known_labels.csv` and `.json`: the package's known-label-only metrics. +- `per_class.csv`: per-class metrics, including unseen labels. +- Logs for each invocation, input SHA-256, pinned revision, exact commands and + a `COMPLETED` marker after all three invocations succeed. + +The package supplies vocabulary coverage and micro/macro statistics. Preserve +both coverage and known-only results when assessing the external dataset; do +not describe known-only accuracy as accuracy on the entire benchmark. Compare +species, genus and family separately. The current package disables its separate +hierarchy-wide metrics; per-level metrics are still produced normally. + +No optimal-threshold fitting, subsampling or label filtering is enabled. The +thresholds recorded in prediction files are retained. These collector files +contain the selected prediction per level, so they cannot establish top-5 +accuracy. The expert dataset evaluates external performance; it should not be +used for selecting this model's thresholds or checkpoints. + +Aggregate metric tables and progress are printed live in the terminal and retained +in the corresponding logs. `pipefail` preserves failures through `tee`. + +For compact inspection after completion: + +```bash +cat /work/evaluation-1/expert/all_labels.csv +cat /work/evaluation-1/expert/known_labels.csv +cat /work/evaluation-1/test/all_labels.csv +``` + +Keep per-class output as a file rather than pasting it into the terminal. +All outputs remain under `/work`; computation can also be rerun on a CPU job +from the saved prediction CSVs without retaining the source images or GPU node. + +## Regional candidate vocabulary + +Both `mt_predict` and `mt_hpredict` accept `--class-list FILE` (YAML key +`class_list`). Supply one exact model class label per UTF-8 line; for GBIF +hierarchical models these are species IDs. Blank lines and duplicates are +ignored. The list restricts the model's current candidate vocabulary and cannot +add species absent from the checkpoint. Missing requested labels are reported; +an empty overlap fails before loading images. + +Every input image and ground-truth label remains in evaluation, including labels +excluded by the list. Species outputs and parent mappings are filtered together. +`class_filter.json` in the prediction output records retained, excluded and +missing labels, plus the list's SHA-256. With a pre-masked checkpoint, filtering +further restricts its active vocabulary rather than restoring excluded classes. + +For example, in an environment containing this CLI feature: + +```bash +mt_hpredict --config /work/expert-full-2/inference.yaml \ + --class-list /work/regional-evaluation/mambo-v2-reduced.txt \ + --output /work/regional-evaluation --name predictions +``` + +Use the all-label report for global versus regional comparisons on the same +images. `known_label` now describes the active filtered vocabulary. Confidence +scores are computed over the restricted candidates, so thresholds calibrated +for the global model should not be assumed equivalent. diff --git a/dev/ucloud/evaluate-results.sh b/dev/ucloud/evaluate-results.sh new file mode 100644 index 00000000..82aa1fdd --- /dev/null +++ b/dev/ucloud/evaluate-results.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# CPU evaluation of completed prediction CSVs, independent of the training venv. +set -euo pipefail +selection="${1:-all}" +output="${2:-/work/evaluation-1}" +case "$selection" in + expert) datasets=(expert) ;; + test) datasets=(test) ;; + all) datasets=(expert test) ;; + *) echo 'Usage: evaluate-results.sh [expert|test|all] [fresh-output-directory]' >&2; exit 2 ;; +esac +revision=70cc69adc05362863439277048e06386c1f885e1 +package="git+https://github.com/GuillaumeMougeot/mini_metrics.git@$revision" +expert_csv="${MT_EXPERT_CSV:-/work/expert-full-2/predictions/mini_metric.csv}" +test_csv="${MT_TEST_CSV:-/work/test-full-1/predictions/mini_metric.csv}" +for dataset in "${datasets[@]}"; do + if [[ "$dataset" == expert ]]; then input="$expert_csv"; else input="$test_csv"; fi + if [[ ! -s "$input" ]]; then + echo "Missing or empty completed prediction file: $input. Finish inference first." >&2 + exit 1 + fi + if [[ -e "$output/$dataset" ]]; then + echo "Output already exists: $output/$dataset; choose a fresh output directory." >&2 + exit 1 + fi +done +export PYTHONUNBUFFERED=1 CUDA_VISIBLE_DEVICES="" OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MPLBACKEND=Agg +runner=(uvx --python 3.13 --from "$package" mm_metrics) +# Resolve the isolated CPU tool once before starting reports. +"${runner[@]}" --help > /dev/null +for dataset in "${datasets[@]}"; do + if [[ "$dataset" == expert ]]; then input="$expert_csv"; else input="$test_csv"; fi + target="$output/$dataset" + mkdir -p "$output" + mkdir "$target" + sha256sum "$input" > "$target/input.sha256" + printf 'mini_metrics_revision=%s\ninput=%s\n' "$revision" "$input" > "$target/provenance.txt" + for scope in all_labels known_labels per_class; do + options=() + case "$scope" in + all_labels) options=(--all) ;; + known_labels) options=(--all --known-only) ;; + per_class) options=(--per-class) ;; + esac + command=("${runner[@]}" --files "$input" --output-dir "$target" --output-name "$scope" --precision 10 "${options[@]}") + printf '%q ' "${command[@]}" >> "$target/commands.sh" + printf '\n' >> "$target/commands.sh" + echo "$dataset / $scope: $target/$scope.log" + if ! "${command[@]}" 2>&1 | tee "$target/$scope.log"; then + echo "Evaluation failed; inspect $target/$scope.log" >&2 + exit 1 + fi + done + touch "$target/COMPLETED" + echo "Completed $dataset evaluation: $target" +done diff --git a/dev/ucloud/experiment.json b/dev/ucloud/experiment.json new file mode 100644 index 00000000..3afc7039 --- /dev/null +++ b/dev/ucloud/experiment.json @@ -0,0 +1,37 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-expanded-nofigures-1", + "environments": { + "master": { + "python": "/work/venvs/mt-master/bin/python", + "commit": "2ebbe39f2365bfea482d7ebc3d000cf4d2044918" + }, + "quant": { + "python": "/work/venvs/mt-quant/bin/python", + "commit": "4ea6b9613fc29a8e33ef72f28885f4e2a0e5906b" + } + }, + "gpus": 1, + "global_batch_size": 32, + "num_workers_per_rank": 2, + "epochs": 4, + "size": 384, + "seeds": [ + 42 + ], + "variants": [ + "master_eager", + "quant_eager", + "quant_prefetch" + ], + "qualification": { + "seed": 42, + "train": 4096, + "validation": 1024, + "test": 128 + }, + "timeout_seconds": 300, + "budget_seconds": 1800, + "require_finite_losses": true, + "figures": false +} diff --git a/dev/ucloud/expert-trial.md b/dev/ucloud/expert-trial.md new file mode 100644 index 00000000..4b3629dd --- /dev/null +++ b/dev/ucloud/expert-trial.md @@ -0,0 +1,84 @@ +# Bounded expert inference staging trial + +Stop the stalled full-test prediction and obsolete expert index creation with +Ctrl+C in their terminals first. Do not run this alongside additional cold-read +jobs. Saved training weights are unaffected; interrupted prediction currently +loses in-memory predictions because its collector writes at completion. + +This helper targets the mounted expert folder and trained weights used in the +current qualification. Run from the node after pulling the committed helpers: + +```bash +cd /work/mini_trainer +git pull --ff-only +bash dev/ucloud/expert-trial.sh +``` + +No manual YAML editing or dataset index construction is required. The launcher +extracts the focused discovery fix at `0c572ca` into a fresh `/work` source overlay +and uses the existing `/work/venvs/mt-quant` interpreter and dependencies. It does +not install packages or update the environment of another process. The underlying +Python helper also accepts explicit source, weights and output paths (`--help`). + +Defaults: + +- GPU 1; 1,024 JPEG/PNG candidates selected round-robin across class folders. +- At most 2 GiB of encoded source images, copied with four readers to + `/dev/shm/mt-expert-expert-staging-trial-1`. +- Five-minute staging limit and ten-minute inference limit. Timeout/interrupt + stops the phase process group, including inference loader workers. +- Normal prediction defaults: model/preprocessing metadata from the weights, + no separately supplied class mapping, resize recipe or fabricated labels. +- Persistent output: `/work/expert-staging-trial-1`. The minimal `inference.yaml` + contains only input and weights; output/name are explicit CLI arguments. + +`/dev/shm` is RAM-backed. The byte cap limits copied images, not total process or +loader memory. This assumes the current large-memory allocation. `/tmp` overlay +storage has not been established as node-local and is not used for this trial. + +Watch either phase in another terminal: + +```bash +tail -f /work/expert-staging-trial-1/stage.log +# Once inference starts: +tail -f /work/expert-staging-trial-1/inference.log +``` + +The launcher prints phase durations. On success, predictions are in +`/work/expert-staging-trial-1/predictions/mini_metric.csv`; `staging.json` records +original and staged paths, bytes and staging duration. Originals are unchanged. +Newly encountered hierarchical labels may still need GBIF/cache resolution; the +existing policy fails explicitly if it cannot resolve ancestors. Removing index +creation does not remove that lookup. + +If either phase times out, inspect its log before retrying. Partial staging is +retained without a completed manifest and is never treated as ready input. A retry +uses a fresh output/staging name, for example: + +```bash +bash dev/ucloud/expert-trial.sh /work/expert-staging-trial-2 --images 256 +``` + +Proceed to larger staging only if this trial completes and staged inference is +fast. Measure the full expert dataset's encoded size and available job memory +before raising the cap. This round-robin subset is a storage/functionality trial, +not the expert benchmark; do not report its accuracy as a full-dataset result. +Retain the full benchmark's unknown species when subsequently running mini_metrics. +The full test split remains deferred until throughput is adequate. + +RAM staging disappears with the job. Predictions, logs, configuration and manifest +are retained under `/work`. Staged files can be deleted once their corresponding +inference has exited; the helper deliberately does not delete existing directories. + +To retry inference after a code fix, reuse completed staging without another read +of the source images: + +```bash +bash dev/ucloud/expert-trial.sh /work/expert-staging-trial-4 \ + --reuse-stage /work/expert-staging-trial-3 +``` + +This checks the previous completed manifest and staged file sizes. The source +paths remain recorded, and the new output receives its own configuration and logs. +Taxonomy still uses the node's existing GBIF response cache; cache/API failures +remain distinct from the corrected count-versus-rank selection bug. diff --git a/dev/ucloud/expert-trial.sh b/dev/ucloud/expert-trial.sh new file mode 100644 index 00000000..ad01da43 --- /dev/null +++ b/dev/ucloud/expert-trial.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Run the bounded expert trial without changing the installed training environment. +set -euo pipefail +repo=$(git -C "$(dirname -- "${BASH_SOURCE[0]}")" rev-parse --show-toplevel) +revision=0c572ca +python=/work/venvs/mt-quant/bin/python +source_dir=/work/flemming_helsing/restructured/valid/referenced +weights=/work/results/global_lepi_production_w32_1/weights/best.pt +output="${1:-/work/expert-staging-trial-1}" +if (( $# > 0 )); then shift; fi +# An isolated source overlay keeps the exact trained package dependencies intact. +git -C "$repo" cat-file -e "$revision:mini_trainer/data/metadata.py" +code=$(mktemp -d /work/mt-inference-code.XXXXXX) +git -C "$repo" archive "$revision" mini_trainer | tar -x -C "$code" +export PYTHONPATH="$code" +exec "$python" "$repo/dev/ucloud/expert_trial.py" \ + --source "$source_dir" --weights "$weights" --output "$output" "$@" diff --git a/dev/ucloud/expert_trial.py b/dev/ucloud/expert_trial.py new file mode 100644 index 00000000..a5bcc019 --- /dev/null +++ b/dev/ucloud/expert_trial.py @@ -0,0 +1,217 @@ +"""Bounded RAM-staging trial followed by the normal hierarchical prediction CLI.""" + +import argparse +import concurrent.futures +import inspect +import json +import os +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +SUFFIXES = {".jpg", ".jpeg", ".png"} + + +def select_images(source: Path, limit: int) -> list[Path]: + """Select round-robin across class folders; this is a trial, not a benchmark.""" + folders = sorted(path for path in source.iterdir() if path.is_dir() and not path.name.startswith(".")) + streams = [iter(path.rglob("*")) for path in folders] + selected = [] + while streams and len(selected) < limit: + active = [] + for stream in streams: + path = next((p for p in stream if p.suffix.lower() in SUFFIXES and p.is_file()), None) + if path is not None: + selected.append(path) + active.append(stream) + if len(selected) == limit: + break + streams = active + return selected + + +def test_rows(index: Path) -> dict: + """Preserve labels and ordering from exactly the supplied test split.""" + data = json.loads(index.read_text()) + count = len(data["path"]) + if any(len(data[key]) != count for key in ("split", "label")): + raise ValueError("Index path, split and label lengths differ") + selected = [i for i, split in enumerate(data["split"]) if split == "test"] + if not selected: + raise ValueError("No test rows in saved index") + result = {k: [v[i] for i in selected] for k, v in data.items() if isinstance(v, list) and len(v) == count} + result["path"] = [str((index.parent / p).absolute()) for p in result["path"]] + return result + + +def stage(source: Path, destination: Path, output: Path, limit: int, byte_limit: int, workers: int, data_index: Path | None = None) -> None: + start = time.monotonic() + print( + f"Selecting all saved test rows from {data_index}" + if data_index + else f"Selecting up to {limit} JPEG/PNG files across class folders", + flush=True, + ) + index = test_rows(data_index) if data_index is not None else None + paths = select_images(source, limit) if index is None else [Path(p) for p in index["path"]] + print(f"Inspecting sizes of {len(paths)} selected files", flush=True) + if not paths: + raise RuntimeError("No JPEG/PNG candidates found") + sizes = [path.stat().st_size for path in paths] + total = sum(sizes) + if total > byte_limit: + raise RuntimeError(f"Selected files need {total} bytes; limit is {byte_limit}. Adjust the staging budget or subset size.") + if shutil.disk_usage(destination.parent).free < total + 256 * 2**20: + raise RuntimeError("Insufficient staging space with 256 MiB headroom") + destination.mkdir(exist_ok=False) + + def relative_path(i: int, path: Path) -> Path: + return path.relative_to(source) if index is None else Path(str(i // 4096)) / f"{i}{path.suffix}" + + def copy(item: tuple[int, Path]) -> dict: + i, path = item + relative = relative_path(i, path) + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + before = path.stat() + shutil.copyfile(path, target) + after = path.stat() + if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns) or target.stat().st_size != before.st_size: + raise RuntimeError(f"Source changed or copy was incomplete: {path}") + return {"source": str(path), "staged": str(target), "bytes": before.st_size} + + rows = [] + print(f"Copying {len(paths)} files ({total / 2**20:.1f} MiB), {workers} readers", flush=True) + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + remaining = iter(enumerate(paths)) + pending = set() + exhausted = False + while pending or not exhausted: + while not exhausted and len(pending) < workers * 2: + item = next(remaining, None) + if item is None: + exhausted = True + else: + pending.add(pool.submit(copy, item)) + if not pending: + break + done, pending = concurrent.futures.wait(pending, return_when=concurrent.futures.FIRST_COMPLETED) + for future in done: + rows.append(future.result()) + if len(rows) % 4096 == 0 or len(rows) == len(paths): + print(f"Copied {len(rows)}/{len(paths)}; elapsed {time.monotonic() - start:.1f}s", flush=True) + report = {"scope": "staging_trial_subset", "seconds": time.monotonic() - start, "bytes": total, "files": rows} + if index is not None: + index["path"] = [str(destination / relative_path(i, p)) for i, p in enumerate(paths)] + (output / "staged-index.json").write_text(json.dumps(index) + "\n") + report["scope"] = "supplied_test_split" + report["source_index"] = str(data_index) + (output / "staging.json").write_text(json.dumps(report, indent=2) + "\n") + + +def reuse_stage(previous: Path) -> tuple[Path, dict]: + """Require a completed manifest and intact staged files, without source reads.""" + config = json.loads((previous / "inference.yaml").read_text()) + report = json.loads((previous / "staging.json").read_text()) + destination = Path(config["input"]).resolve(strict=True) + rows = report["files"] + if not rows: + raise RuntimeError("Completed staging manifest contains no files") + for row in rows: + path = Path(row["staged"]).resolve(strict=True) + if not path.is_relative_to(destination) or path.stat().st_size != row["bytes"]: + raise RuntimeError(f"Staged file does not match manifest: {path}") + return destination, report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--weights", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True, help="Fresh persistent trial directory") + parser.add_argument("--images", type=int, default=1024) + parser.add_argument("--data-index", type=Path, help="Stage all test rows from this saved index; ignores --images") + parser.add_argument("--max-mib", type=int, default=2048) + parser.add_argument("--copy-workers", type=int, default=4) + parser.add_argument("--stage-timeout", type=int, default=300) + parser.add_argument("--inference-timeout", type=int, default=600) + parser.add_argument("--gpu", default="1") + parser.add_argument("--reuse-stage", type=Path, help="Reuse a completed trial directory without copying source images") + parser.add_argument("--stage-only", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + if min(args.images, args.max_mib, args.copy_workers, args.stage_timeout, args.inference_timeout) <= 0: + parser.error("Limits and worker counts must be positive") + args.source = args.source.resolve(strict=True) + args.weights = args.weights.resolve(strict=True) + args.output = args.output.resolve() + destination = Path("/dev/shm") / ("mt-expert-" + args.output.name) + if args.stage_only: + stage(args.source, destination, args.output, args.images, args.max_mib * 2**20, args.copy_workers, args.data_index) + return + + from mini_trainer.data import auto_find_images + + if "cls2idx" not in inspect.signature(auto_find_images).parameters: + raise RuntimeError("Install the focused inference discovery fix before running this trial") + previous_report = None + if args.reuse_stage is not None: + destination, previous_report = reuse_stage(args.reuse_stage) + elif destination.exists(): + raise RuntimeError(f"Staging directory already exists: {destination}; choose a fresh output name") + args.output.mkdir(parents=True, exist_ok=False) + if previous_report is not None: + (args.output / "staging.json").write_text(json.dumps(previous_report, indent=2) + "\n") + print(f"Reusing completed staging from {args.reuse_stage}: {destination}", flush=True) + config = {"input": str(destination), "weights": str(args.weights)} + if args.data_index is not None: + if args.reuse_stage is not None: + shutil.copyfile(args.reuse_stage / "staged-index.json", args.output / "staged-index.json") + config["data_index"] = str(args.output / "staged-index.json") + config_path = args.output / "inference.yaml" + # JSON is valid YAML; no extra dependency or hand-edited config required. + config_path.write_text(json.dumps(config, indent=2) + "\n") + env = dict(os.environ, CUDA_VISIBLE_DEVICES=args.gpu, OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", MPLBACKEND="Agg") + cli = Path(sys.executable).parent / "mt_hpredict" + if not cli.is_file(): + raise RuntimeError(f"Missing installed CLI: {cli}") + commands = [ + ([sys.executable, str(Path(__file__).resolve()), *sys.argv[1:], "--stage-only"], args.stage_timeout, "stage"), + ( + [str(cli), "--config", str(config_path), "--output", str(args.output), "--name", "predictions"], + args.inference_timeout, + "inference", + ), + ] + if args.reuse_stage is not None: + commands = commands[1:] + for command, timeout, phase in commands: + print(f"Starting {phase}; timeout {timeout}s. Log: {args.output / (phase + '.log')}", flush=True) + started = time.monotonic() + with (args.output / f"{phase}.log").open("w") as log: + with subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) as proc: + try: + returncode = proc.wait(timeout=timeout) + except (subprocess.TimeoutExpired, KeyboardInterrupt): + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + # Also stop loader descendants, even if the parent exited first. + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + raise SystemExit(f"{phase} interrupted or exceeded {timeout}s; see {log.name}.") from None + if returncode: + raise SystemExit(f"{phase} failed ({returncode}); see {log.name}") + print(f"{phase} completed in {time.monotonic() - started:.1f}s", flush=True) + print(f"Trial complete: {args.output}. Staged bytes retained at {destination}.", flush=True) + print("Check staging.json for the selected scope and exact file count.", flush=True) + + +if __name__ == "__main__": + main() diff --git a/dev/ucloud/export_followup.py b/dev/ucloud/export_followup.py new file mode 100644 index 00000000..ab4ec4e8 --- /dev/null +++ b/dev/ucloud/export_followup.py @@ -0,0 +1,88 @@ +"""Export a completed floating-point run and retain real validation inputs for ORT.""" + +import argparse +import json +from pathlib import Path + +from compare import digest, write_json + + +def main(): + import numpy as np + import onnxruntime as ort + import torch + + from mini_trainer.data.io import make_read_and_resize_fn + from mini_trainer.modeling import Classifier + from mini_trainer.modeling.onnx import export_onnx + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("comparison", type=Path) + parser.add_argument("run") + args = parser.parse_args() + root = args.comparison.resolve() + run = root / "runs" / args.run + result = json.loads((run / "result.json").read_text()) + if result["status"] != "completed" or digest(result["checkpoint"]) != result["checkpoint_sha256"]: + raise ValueError("Require a completed run with an unchanged checkpoint") + if result["options"].get("quantized_training"): + raise ValueError("This FP32 follow-up excludes native INT8; use the explicit native INT8 export workflow in docs/onnx.md") + prepared = json.loads((root / "prepared.json").read_text()) + if digest(root / "data_index.json") != prepared["data_index.json"]: + raise ValueError("Frozen dataset index changed") + config = json.loads((root / "comparison.json").read_text()) + data = json.loads((root / "data_index.json").read_text()) + paths = [p for p, split in zip(data["path"], data["split"], strict=True) if split == "validation"][:4] + if len(paths) < 4: + raise ValueError("Need four real validation images for dynamic-batch verification") + model, preprocess = Classifier.build( + weights=result["checkpoint"], + device="cpu", + dtype=torch.float32, + model_args={"pretrained": False}, + ) + model.eval() + reader = make_read_and_resize_fn((config["size"], config["size"]), torch.device("cpu"), torch.uint8) + with torch.inference_mode(): + images = preprocess(torch.stack([reader(p) for p in paths])) + eager = model(images) + destination = run / "onnx" + export_onnx( + model, + images[:2], + destination, + verification_inputs=[images[:1], images], + preprocessing={ + "loader": "mini_trainer.data.io.make_read_and_resize_fn", + "size": [config["size"], config["size"]], + "loader_dtype": "uint8", + "model_preprocess": repr(preprocess), + "example_image_sha256": {p: digest(p) for p in paths}, + "description": "RGB decode and repository resize, then checkpoint preprocessor; preprocessing remains outside ONNX", + }, + checkpoint_sha256=result["checkpoint_sha256"], + ) + manifest = json.loads((destination / "manifest.json").read_text()) + session = ort.InferenceSession(str(destination / "model.onnx"), providers=["CPUExecutionProvider"]) + outputs = session.run(None, {manifest["input"]["name"]: images.numpy()}) + if len(outputs) != len(eager): + raise ValueError("Output count differs") + for expected, actual in zip(eager, outputs, strict=True): + np.testing.assert_allclose(actual, expected.numpy(), rtol=1e-4, atol=1e-5) + np.savez(destination / "validation-example.npz", images=images.numpy(), **{f"output_{i}": v for i, v in enumerate(outputs)}) + write_json( + destination / "real-image-parity.json", + { + "status": "passed", + "images": paths, + "checkpoint_sha256": result["checkpoint_sha256"], + "provider": "CPUExecutionProvider", + "rtol": 1e-4, + "atol": 1e-5, + "scope": "Four validation images; dynamic-batch FP32 export parity, not held-out quality or target-GPU speed", + }, + ) + + +if __name__ == "__main__": + main() diff --git a/dev/ucloud/figures.json b/dev/ucloud/figures.json new file mode 100644 index 00000000..8a91742e --- /dev/null +++ b/dev/ucloud/figures.json @@ -0,0 +1,37 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-figures-3", + "environments": { + "master": { + "python": "/work/venvs/mt-master/bin/python", + "commit": "2ebbe39f2365bfea482d7ebc3d000cf4d2044918" + }, + "quant": { + "python": "/work/venvs/mt-quant/bin/python", + "commit": "42c0f065e899ef6866bcf1c9e1304251824a9d8a" + } + }, + "gpus": 1, + "global_batch_size": 32, + "num_workers_per_rank": 2, + "epochs": 2, + "size": 384, + "seeds": [ + 42 + ], + "variants": [ + "master_eager", + "quant_eager", + "quant_compile_model" + ], + "qualification": { + "seed": 42, + "train": 4096, + "validation": 1024, + "test": 128 + }, + "timeout_seconds": 300, + "budget_seconds": 1800, + "require_finite_losses": true, + "figures": true +} diff --git a/dev/ucloud/launch.sh b/dev/ucloud/launch.sh new file mode 100644 index 00000000..221b34f5 --- /dev/null +++ b/dev/ucloud/launch.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Run once inside an allocated UCloud GPU job (one node, multiple GPUs). +set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}" +export MKL_NUM_THREADS="${MKL_NUM_THREADS:-1}" +export TORCHINDUCTOR_COMPILE_THREADS="${TORCHINDUCTOR_COMPILE_THREADS:-1}" +export MPLBACKEND=Agg +export PYTHONHASHSEED=0 +export TORCH_NCCL_ASYNC_ERROR_HANDLING=1 +# Use the fresh training environment even when the node's system Python is older. +if (( $# == 0 )); then + echo 'Usage: bash launch.sh comparison.json [--stage plan|prepare|train|summary] [--only RUN]' >&2 + exit 2 +fi +controller_python=${CONTROLLER_PYTHON:-} +if [[ -z "$controller_python" ]]; then + controller_python=$(python3 -c 'import json, os, sys; print(os.path.expanduser(json.load(open(sys.argv[1]))["environments"]["quant"]["python"]))' "$1") +fi +exec "$controller_python" "$script_dir/compare.py" "$@" diff --git a/dev/ucloud/optimizer-compilation.json b/dev/ucloud/optimizer-compilation.json new file mode 100644 index 00000000..4a346d68 --- /dev/null +++ b/dev/ucloud/optimizer-compilation.json @@ -0,0 +1,37 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-compile-optimizer-1", + "environments": { + "master": { + "python": "/work/venvs/mt-master/bin/python", + "commit": "2ebbe39f2365bfea482d7ebc3d000cf4d2044918" + }, + "quant": { + "python": "/work/venvs/mt-quant/bin/python", + "commit": "4ea6b9613fc29a8e33ef72f28885f4e2a0e5906b" + } + }, + "gpus": 1, + "global_batch_size": 32, + "num_workers_per_rank": 2, + "epochs": 4, + "size": 384, + "seeds": [ + 42 + ], + "variants": [ + "master_eager", + "quant_eager", + "quant_compile_optimizer" + ], + "qualification": { + "seed": 42, + "train": 4096, + "validation": 1024, + "test": 128 + }, + "timeout_seconds": 300, + "budget_seconds": 1800, + "require_finite_losses": true, + "figures": false +} diff --git a/dev/ucloud/production.py b/dev/ucloud/production.py new file mode 100644 index 00000000..266229db --- /dev/null +++ b/dev/ucloud/production.py @@ -0,0 +1,118 @@ +"""Generate a production CLI config from verified full-taxonomy qualification artifacts.""" + +import argparse +import hashlib +import json +from pathlib import Path + +import yaml + +STATISTICS = [ + "acc1", + "acc5", + "loss", + "lr", + "item/s", + "mem", + "step", + "time", + "eta", + "epoch", + "type", + "acc1/lvl1", + "acc5/lvl1", + "acc1/lvl2", + "acc5/lvl2", + "loss/lvl0", + "loss/lvl1", + "loss/lvl2", +] + + +def generate(root, destination, output, name, batch, workers, epochs, project): + root, destination, output = Path(root).resolve(), Path(destination).resolve(), Path(output).resolve() + if min(batch, epochs) < 1 or workers < 0: + raise ValueError("Positive batch/epochs and nonnegative workers required") + if not name or Path(name).name != name or name in (".", ".."): + raise ValueError("Run name must be a single directory name") + config = json.loads((root / "comparison.json").read_text()) + if config.get("mode") != "scaling" or not config.get("full_taxonomy") or config["gpus"] != 4: + raise ValueError("Require full-taxonomy four-GPU scaling preparation") + manifest = json.loads((root / "prepared.json").read_text()) + seed = config["seeds"][0] + for filename in ("class_spec.json", "dataset.json", f"initial_seed{seed}.pt"): + with (root / filename).open("rb") as handle: + actual = hashlib.file_digest(handle, "sha256").hexdigest() + if actual != manifest[filename]: + raise ValueError(f"Prepared artifact changed: {filename}") + if (output / name).exists(): + raise ValueError("Production output already exists; use a new run name or reviewed resume procedure") + settings = { + "input": config["parquet"], + "output": str(output), + "name": name, + "class_spec": str(root / "class_spec.json"), + "epochs": epochs, + "size": config["size"], + "seed": seed, + "device": "cuda", + "dtype": "float16", + "ema": False, + "compile": True, + "compile_optimizer": False, + "quantized_training": False, + "model_builder_kwargs": { + "model_type": "efficientnet_v2_s", + "weights": str(root / f"initial_seed{seed}.pt"), + "model_args": {"pretrained": False}, + "hidden": True, + "normalized": True, + "droprate": 0.1, + "fine_tune": False, + }, + "dataloader_builder_kwargs": { + "batch_size": batch, + "num_workers": workers, + "cache": None, + "splits": ["train", "val"], + "resample": False, + "cuda_prefetch": False, + }, + "optimizer_builder_kwargs": {"optimizer_cls": "mini_trainer.training.MuonAuxAdamW", "lr": 0.001, "weight_decay": 0.01}, + "criterion_builder_kwargs": {"weighted": True, "weights": [1, 1, 1]}, + "regularizer_builder_kwargs": {"strength": 0.1}, + "lr_schedule_builder_kwargs": {"warmup_epochs": 0.25}, + "logger_builder_kwargs": { + "verbose": True, + "statistics": STATISTICS, + "logger_cls_extra_kwargs": [ + {}, + {"project": project, "run_name": name, "run_id": hashlib.sha256(str(output / name).encode()).hexdigest()[:16]}, + ], + }, + } + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("x") as handle: + yaml.safe_dump(settings, handle, sort_keys=False) + return settings + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("qualification", type=Path) + parser.add_argument("destination", type=Path) + parser.add_argument("--output", type=Path, required=True, help="Persistent production output parent") + parser.add_argument("--name", required=True) + parser.add_argument("--batch", type=int, required=True, help="Per GPU") + parser.add_argument("--workers", type=int, required=True, help="Per GPU") + parser.add_argument("--epochs", type=int, required=True, help="Reviewed full-dataset scheduler horizon") + parser.add_argument("--project", default="mini-trainer-production") + args = parser.parse_args() + config = generate(args.qualification, args.destination, args.output, args.name, args.batch, args.workers, args.epochs, args.project) + print(f"Wrote {args.destination}; global batch={4 * args.batch}, epochs={args.epochs}") + print(f"Full source: {config['input']}; no qualification data_index is used") + print("Launch with the pinned environment's mt_htrain --config FILE --wandb --compile under four-rank torchrun") + + +if __name__ == "__main__": + main() diff --git a/dev/ucloud/qualification.json b/dev/ucloud/qualification.json new file mode 100644 index 00000000..7b72a92f --- /dev/null +++ b/dev/ucloud/qualification.json @@ -0,0 +1,18 @@ +{ + "parquet": "/work/global_lepi/0032836-250426092105405_processing_metadata_postprocessed_quality_filtered.parquet", + "output": "/work/results/global-lepi-smoke-1", + "environments": { + "master": {"python": "/work/venvs/mt-master/bin/python", "commit": "2ebbe39f2365bfea482d7ebc3d000cf4d2044918"}, + "quant": {"python": "/work/venvs/mt-quant/bin/python", "commit": "4ea6b9613fc29a8e33ef72f28885f4e2a0e5906b"} + }, + "gpus": 1, + "global_batch_size": 32, + "num_workers_per_rank": 2, + "epochs": 1, + "size": 384, + "seeds": [42], + "variants": ["master_eager", "quant_eager"], + "qualification": {"seed": 42, "train": 2048, "validation": 512, "test": 128}, + "timeout_seconds": 1800, + "budget_seconds": 1800 +} diff --git a/dev/ucloud/replay_validation.py b/dev/ucloud/replay_validation.py new file mode 100644 index 00000000..4591699d --- /dev/null +++ b/dev/ucloud/replay_validation.py @@ -0,0 +1,142 @@ +"""Replay one saved checkpoint's validation batches; diagnostic, not a benchmark.""" + +import argparse +import hashlib +import json +from pathlib import Path + +from compare import digest, write_json + + +def replay(config, data, spec, checkpoint, *, precision, prefetch, output, device="cuda"): + import torch + + from mini_trainer.data.loader import get_dataset_dataloader + from mini_trainer.hierarchical.integration import HierarchicalBuilder + from mini_trainer.modeling import Classifier + + device = torch.device(device) + dtype = torch.float16 if precision == "fp16" else torch.float32 + model, preprocess = Classifier.build(weights=checkpoint, device=device, dtype=torch.float32, model_args={"pretrained": False}) + model.eval() + parameters_finite = all(bool(torch.isfinite(p).all()) for p in model.parameters()) + nonfinite_buffers = [name for name, value in model.named_buffers() if not bool(torch.isfinite(value).all())] + labels = [label for label, split in zip(data["class"], data["split"], strict=True) if split == "train"] + criterion = HierarchicalBuilder.build_criterion( + num_classes=spec["num_classes"], + labels=labels, + device=device, + dtype=dtype, + weighted=True, + label_smoothing=None, + weights=[1, 1, 1], + ) + selected = [i for i, split in enumerate(data["split"]) if split == "validation"] + metadata = {key: [data[key][i] for i in selected] for key in ("path", "class")} + _, loaders = get_dataset_dataloader( + metadata, + resize_size=(config["size"], config["size"]), + modes=("val",), + batch_size=config["global_batch_size"], + num_workers=config["num_workers_per_rank"], + device=device, + dtype=dtype, + cache=None, + multilabel=True, + cuda_prefetch=prefetch, + ) + records = [] + offset = 0 + with (output / "batches.jsonl").open("w") as handle: + for index, (batch, targets) in enumerate(loaders[0]): + batch = batch.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + with torch.inference_mode(), torch.autocast(device.type, dtype=dtype, enabled=precision == "fp16"): + images = preprocess(batch) + logits = model(images) + losses = criterion(logits, targets) + # Inspect only after inference has consumed the asynchronous batch. + # These synchronizations can still hide timing-sensitive failures. + finite = { + "input": bool(torch.isfinite(images).all()), + "outputs": [bool(torch.isfinite(value).all()) for value in logits], + "losses": [bool(torch.isfinite(value).all()) for value in losses], + } + values = [float(value) for value in losses] + record = { + "batch": index, + "paths": metadata["path"][offset : offset + len(batch)], + "input_sha256": hashlib.sha256(batch.cpu().contiguous().numpy().tobytes()).hexdigest(), + "targets_sha256": hashlib.sha256(targets.cpu().contiguous().numpy().tobytes()).hexdigest(), + "finite": finite, + "losses": [value if good else str(value) for value, good in zip(values, finite["losses"], strict=True)], + } + handle.write(json.dumps(record) + "\n") + handle.flush() + records.append(record) + offset += len(batch) + print(f"batch={index} finite={finite}", flush=True) + passed = ( + bool(records) + and parameters_finite + and not nonfinite_buffers + and all(r["finite"]["input"] and all(r["finite"]["outputs"]) and all(r["finite"]["losses"]) for r in records) + ) + result = { + "status": "passed" if passed else "nonfinite", + "precision": precision, + "prefetch": prefetch, + "parameters_finite": parameters_finite, + "nonfinite_buffers": nonfinite_buffers, + "torch": torch.__version__, + "cuda": torch.version.cuda, + "device": str(device), + "preprocessing": repr(preprocess), + "criterion": repr(criterion), + "batches": len(records), + "images": offset, + "checkpoint_sha256": digest(checkpoint), + "scope": "Reloaded checkpoint validation with per-batch synchronization; no optimizer, training or timing comparison", + } + write_json(output / "result.json", result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("comparison", type=Path) + parser.add_argument("run") + parser.add_argument("--precision", choices=("fp16", "fp32"), required=True) + parser.add_argument("--prefetch", action="store_true") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + root = args.comparison.resolve() + config = json.loads((root / "comparison.json").read_text()) + if config["gpus"] != 1: + raise ValueError("This diagnostic requires the single-GPU qualification") + result = json.loads((root / "runs" / args.run / "result.json").read_text()) + checkpoint = result["checkpoint"] + if digest(checkpoint) != result["checkpoint_sha256"]: + raise ValueError("Checkpoint changed") + prepared = json.loads((root / "prepared.json").read_text()) + for name in ("data_index.json", "class_spec.json"): + if digest(root / name) != prepared[name]: + raise ValueError(f"Frozen artifact changed: {name}") + args.output.mkdir(parents=True, exist_ok=False) + write_json( + args.output / "source.json", + {"comparison": str(root), "config_sha256": digest(root / "comparison.json"), "run": args.run, "checkpoint": checkpoint}, + ) + replay( + config, + json.loads((root / "data_index.json").read_text()), + json.loads((root / "class_spec.json").read_text()), + checkpoint, + precision=args.precision, + prefetch=args.prefetch, + output=args.output, + ) + + +if __name__ == "__main__": + main() diff --git a/dev/ucloud/scaling.py b/dev/ucloud/scaling.py new file mode 100644 index 00000000..2419f2c7 --- /dev/null +++ b/dev/ucloud/scaling.py @@ -0,0 +1,137 @@ +"""Generate bounded DDP trials and summarize per-rank throughput/loading evidence.""" + +import argparse +import json +import math +import statistics +from pathlib import Path + +from compare import validate + + +def trial(base, destination, output, batch, epochs, checkpoint=None, resume_epoch=None, workers=None, storage=False): + config = json.loads(Path(base).read_text()) + original_output = config["output"] + config.pop("checkpoint", None) + config.pop("resume_epoch", None) + config.update( + global_batch_size=batch * config["gpus"], epochs=epochs, output=str(Path(output).resolve()), reuse_preparation=original_output + ) + if workers is not None: + config["num_workers_per_rank"] = workers + validate(config) + if storage: + if checkpoint: + raise ValueError("Storage selection cannot be combined with checkpoint restoration") + source = Path(original_output) + manifest = json.loads((source / "prepared.json").read_text()) + config.pop("reuse_preparation") + config.update( + exclude_qualification=str(source / "qualification.parquet"), + exclude_qualification_sha256=manifest["qualification.parquet"], + epochs=1, + timeout_seconds=600, + budget_seconds=1800, + ) + config["qualification"] = dict(config["qualification"], train=262144, validation=8192, test=128) + elif not checkpoint and config.get("qualification"): + steps = config["qualification"]["train"] // config["global_batch_size"] + if steps < 1: + raise ValueError("Need at least one complete global batch") + config["epochs"] = max(epochs, 1 + math.ceil(100 / steps)) + if checkpoint: + config.update(checkpoint=str(Path(checkpoint).resolve()), resume_epoch=resume_epoch) + config = validate(config) + Path(destination).parent.mkdir(parents=True, exist_ok=True) + with Path(destination).open("x") as handle: + json.dump(config, handle, indent=2) + handle.write("\n") + return config + + +def report(root): + root = Path(root) + config = json.loads((root / "comparison.json").read_text()) + environment = json.loads((root / "environment-quant.json").read_text()) + results = [] + for result in sorted(root.glob("runs/*/result.json")): + row = json.loads(result.read_text()) + ranks = [] + for rank in range(config["gpus"]): + path = result.parent / f"phases-rank{rank}.jsonl" + ranks.append([json.loads(line) for line in path.read_text().splitlines()] if path.is_file() else []) + epochs = sorted(set(p["epoch"] for rank in ranks for p in rank if p["phase"] == "train")) + measurements = [] + for epoch in epochs: + phases = [next((p for p in rank if p["phase"] == "train" and p["epoch"] == epoch), None) for rank in ranks] + if any(p is None for p in phases): + continue + seconds = max(p["seconds"] for p in phases) + measurements.append( + { + "epoch": epoch, + "images_per_second": sum(p["samples"] for p in phases) / seconds, + "seconds": seconds, + "rank_time_ratio": seconds / min(p["seconds"] for p in phases), + "steps": min(p.get("steps", 0) for p in phases), + "windows_by_rank": [p.get("windows", []) for p in phases], + "max_loader_wait_seconds": max(p["loader_wait_seconds"] for p in phases), + } + ) + warm = measurements[1:] + peaks = [max((p["max_memory_reserved"] for p in rank), default=0) for rank in ranks] + row.update( + warm_steps=sum(p["steps"] for p in warm), + sufficient_warm_steps=sum(p["steps"] for p in warm) >= 100, + performance_eligible=(row.get("status") == "completed" and all(ranks) and sum(p["steps"] for p in warm) >= 100), + phases_by_rank=ranks, + component_timings_by_rank=[ + [json.loads(line) for line in path.read_text().splitlines()] + for path in sorted(result.parent.glob("components-rank*.jsonl")) + ], + peak_allocated_bytes=max((p.get("max_memory_allocated", 0) for rank in ranks for p in rank), default=0), + per_gpu_batch=config["global_batch_size"] // config["gpus"], + epochs=measurements, + warm_images_per_second=statistics.median(p["images_per_second"] for p in warm) if warm else None, + peak_reserved_fraction=max(p / total for p, total in zip(peaks, environment["gpu_memory_bytes"], strict=True)), + all_ranks_recorded=all(ranks), + restore_checks=len(list(result.parent.glob("restore-rank*.json"))), + ) + results.append(row) + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="action", required=True) + create = commands.add_parser("trial") + create.add_argument("base") + create.add_argument("destination") + create.add_argument("--output", required=True) + create.add_argument("--batch", type=int, required=True, help="Per GPU batch") + create.add_argument("--epochs", type=int, default=3) + create.add_argument("--workers", type=int, help="Loader workers per rank") + create.add_argument("--storage", action="store_true", help="Separate campaign with a larger disjoint selection") + create.add_argument("--checkpoint") + create.add_argument("--resume-epoch", type=int) + summarize = commands.add_parser("report") + summarize.add_argument("root") + args = parser.parse_args() + if args.action == "trial": + trial( + args.base, + args.destination, + args.output, + args.batch, + args.epochs, + args.checkpoint, + args.resume_epoch, + args.workers, + args.storage, + ) + else: + print(json.dumps(report(args.root), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/dev/ucloud/setup.sh b/dev/ucloud/setup.sh new file mode 100644 index 00000000..4c5823bd --- /dev/null +++ b/dev/ucloud/setup.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Dedicated environments for a fresh manually allocated UCloud job; no training. +set -euo pipefail + +if (( $# != 1 )); then + echo 'Usage: bash dev/ucloud/setup.sh /work/DATASET/metadata.parquet' >&2 + exit 2 +fi +for tool in uv git python3; do + command -v "$tool" >/dev/null || { echo "Missing $tool; install it before setup" >&2; exit 2; } +done +[[ -f "$1" ]] || { echo "Parquet file not found: $1" >&2; exit 2; } +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(git -C "$script_dir" rev-parse --show-toplevel) +work_root=${MT_WORK_ROOT:-/work} +mkdir -p "$work_root/venvs" +work_root=$(cd -- "$work_root" && pwd) +node_config=${MT_CONFIG:-$work_root/qualification.json} +node_config=$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).resolve())' "$node_config") +[[ ! -e "$node_config" ]] || { echo "Config already exists: $node_config; preserve it or choose a new MT_CONFIG" >&2; exit 2; } +mkdir -p -- "$(dirname -- "$node_config")" +backend=${MT_TORCH_BACKEND:-cu130} +case "$backend" in + cu126|cu130|cu132) ;; + *) echo 'MT_TORCH_BACKEND must be cu126, cu130 or cu132' >&2; exit 2 ;; +esac +template=${MT_TEMPLATE:-$script_dir/qualification.json} +master_sha=$(python3 -c 'import json,sys; envs=json.load(open(sys.argv[1]))["environments"]; print(envs.get("master", envs["quant"])["commit"])' "$template") +quant_sha=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["environments"]["quant"]["commit"])' "$template") +for sha in "$master_sha" "$quant_sha"; do + git -C "$repo_root" cat-file -e "$sha^{commit}" || { echo "Missing pinned commit $sha; fetch full repository history" >&2; exit 2; } +done +repo_url=${MT_REPO_URL:-https://github.com/asgersvenning/mini_trainer.git} +export UV_CACHE_DIR=${UV_CACHE_DIR:-$work_root/.cache/uv} +export UV_LINK_MODE=copy +unset UV_TORCH_BACKEND + +export_dir=$(mktemp -d) +trap 'rm -rf -- "$export_dir"' EXIT +git -C "$repo_root" archive "$quant_sha" pyproject.toml uv.lock | tar -xf - -C "$export_dir" +requirements="$work_root/requirements-mt.txt" +echo "Exporting pinned dependencies from $quant_sha ($backend)" +uv export --directory "$export_dir" --locked --no-dev --no-emit-project \ + --extra recommended --extra "$backend" --extra export --extra quantization \ + --output-file "$requirements" >/dev/null + +mapfile -t branches < <(python3 -c 'import json,sys; print("\n".join(json.load(open(sys.argv[1]))["environments"]))' "$template") +for branch in "${branches[@]}"; do + env_dir="$work_root/venvs/mt-$branch" + if [[ ! -x "$env_dir/bin/python" ]]; then + uv venv --python 3.12 "$env_dir" + fi + echo "Installing locked dependencies in $env_dir" + # These are dedicated comparison environments. Sync removes leftover packages + # when retrying a partially completed setup; it never touches the node's Python. + uv pip sync --python "$env_dir/bin/python" --require-hashes \ + --index https://pypi.org/simple --default-index "https://download.pytorch.org/whl/$backend" \ + --index-strategy unsafe-first-match "$requirements" + sha=$master_sha + if [[ "$branch" == quant ]]; then sha=$quant_sha; fi + uv pip install --python "$env_dir/bin/python" --no-deps "mini_trainer @ git+$repo_url@$sha" + uv pip check --python "$env_dir/bin/python" +done + +python3 - "$template" "$1" "$work_root" "$node_config" <<'PY' +import json +import sys +from pathlib import Path + +template, parquet, work_root, destination = map(Path, sys.argv[1:]) +config = json.loads(template.read_text()) +config["parquet"] = str(parquet.resolve()) +config["output"] = str(work_root / "results" / Path(config["output"]).name) +for branch, entry in config["environments"].items(): + entry["python"] = str(work_root / "venvs" / f"mt-{branch}" / "bin" / "python") +with destination.open("x") as handle: + json.dump(config, handle, indent=2) + handle.write("\n") +print(f"Setup complete. Node config: {destination}") +PY +printf 'Next: bash %q %q --stage plan\n' "$script_dir/launch.sh" "$node_config" +echo 'Then run prepare followed immediately by train; the 30-minute budget starts at prepare.' diff --git a/dev/ucloud/test-inference.md b/dev/ucloud/test-inference.md new file mode 100644 index 00000000..cff8afc0 --- /dev/null +++ b/dev/ucloud/test-inference.md @@ -0,0 +1,41 @@ +# Full in-domain test inference + +From `/work/mini_trainer`, run in tmux: + +```bash +git pull --ff-only +bash dev/ucloud/test-inference.sh /work/test-full-1 +``` + +This selects every `test` row from the production `data_index.json`, preserves +labels, class indices and ordering, and copies encoded files into `/dev/shm` +with 512 I/O threads and bounded pending work. It does not select training or +validation rows, resize images, or resolve taxonomy. The 128 GiB staging cap +and free-space check apply before copying. Size inspection precedes copy progress. + +The normal `mt_hpredict` CLI runs on GPU 0 with input, weights and the staged +index supplied; other defaults come from the inference CLI and model metadata. +The existing isolated inference source overlay is used without reinstalling. +Each stage has a one-hour timeout. Stop any old stalled test inference before +starting this run. This helper does not terminate unrelated processes. + +```bash +tail -n 5 /work/test-full-1/stage.log +tail -n 5 /work/test-full-1/inference.log +``` + +The CSV is `/work/test-full-1/predictions/mini_metric.csv`; configuration, +source-to-staged manifest and logs remain in `/work/test-full-1`. RAM files do +not survive job termination. Completed staging can be reused after an inference +failure, without reading source images again: + +```bash +bash dev/ucloud/test-inference.sh /work/test-full-2 --reuse-stage /work/test-full-1 +``` + +A completed manifest is required for reuse. An interrupted copy is not resumable. +The selected test count should match the production index (expected 632,913 +from the original dataset; the saved index is authoritative). + +After inference, use [the mini_metrics evaluation helper](evaluate-results.md) +for separate in-domain and expert reports. diff --git a/dev/ucloud/test-inference.sh b/dev/ucloud/test-inference.sh new file mode 100644 index 00000000..ec6fffc1 --- /dev/null +++ b/dev/ucloud/test-inference.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Stage the complete saved test split and infer without changing the environment. +set -euo pipefail +repo=$(git -C "$(dirname -- "${BASH_SOURCE[0]}")" rev-parse --show-toplevel) +output="${1:-/work/test-full-1}" +if (( $# > 0 )); then shift; fi +exec bash "$repo/dev/ucloud/expert-trial.sh" "$output" \ + --source /work/global_lepi \ + --data-index /work/results/global_lepi_production_w32_1/data_index.json \ + --copy-workers 512 --max-mib 131072 \ + --stage-timeout 3600 --inference-timeout 3600 --gpu 0 "$@" diff --git a/dev/ucloud/worker.py b/dev/ucloud/worker.py new file mode 100644 index 00000000..2f02a6ae --- /dev/null +++ b/dev/ucloud/worker.py @@ -0,0 +1,555 @@ +"""Branch-neutral worker, executed with the selected installed-package interpreter.""" + +import argparse +import functools +import importlib.metadata +import inspect +import json +import os +import platform +import random +import shutil +import tempfile +import time +from collections import Counter +from pathlib import Path + +from compare import digest, plan, uses_quantized_training, write_json + + +def progress(message): + print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {message}", flush=True) + + +def qualification_parquet(config): + """Reservoir-sample each existing split with memory bounded by sample size.""" + import pyarrow as pa + import pyarrow.parquet as pq + + from mini_trainer.integrations.parquet import COLUMNS, iter_parquet_batches, set2split + + excluded = set() + exclusion = config.get("exclude_qualification") + if exclusion: + if digest(exclusion) != config["exclude_qualification_sha256"]: + raise ValueError("Exclusion sample changed since trial generation") + excluded = { + (str(row["speciesKey"]), row["filename"]) for row in pq.read_table(exclusion, columns=["speciesKey", "filename"]).to_pylist() + } + settings = config["qualification"] + rng = random.Random(settings["seed"]) + selected = {split: [] for split in ("train", "validation", "test")} + seen = Counter() + scanned = 0 + next_report = 1_000_000 + progress("Selecting qualification rows from existing splits (one streaming Parquet scan)") + for batch in iter_parquet_batches(config["parquet"]): + for row in batch.to_pylist(): + if (str(row["speciesKey"]), row["filename"]) in excluded: + continue + split = set2split(int(row["set"])) + seen[split] += 1 + scanned += 1 + reservoir = selected[split] + limit = settings[split] + if len(reservoir) < limit: + reservoir.append((scanned, row)) + else: + index = rng.randrange(seen[split]) + if index < limit: + reservoir[index] = (scanned, row) + if scanned >= next_report: + progress(f"Scanned {scanned:,} included rows; retained {sum(map(len, selected.values())):,}") + next_report = scanned + 1_000_000 + if any(seen[split] < settings[split] for split in selected): + raise ValueError(f"Not enough rows for requested qualification sample; available: {dict(seen)}") + rows = [row for _, row in sorted(item for reservoir in selected.values() for item in reservoir)] + source_schema = pq.read_schema(config["parquet"]) + schema = pa.schema([source_schema.field(column) for column in COLUMNS]) + destination = Path(config["output"]) / "qualification.parquet" + pq.write_table(pa.Table.from_pylist(rows, schema=schema), destination) + if any((str(row["speciesKey"]), row["filename"]) in excluded for row in rows): + raise ValueError("Qualification selection overlaps excluded images") + write_json( + Path(config["output"]) / "selection.json", + { + "sha256": digest(destination), + "excluded_sha256": config.get("exclude_qualification_sha256"), + "excluded_images": len(excluded), + "eligible_split_counts": dict(seen), + }, + ) + progress(f"Saved {len(rows):,} qualification rows; original split counts: {dict(seen)}") + return str(destination) + + +def preflight(config, branch, *, verify=False): + import torch + + import mini_trainer + + distribution = importlib.metadata.distribution("mini_trainer") + direct = json.loads(distribution.read_text("direct_url.json") or "{}") + commit = direct.get("vcs_info", {}).get("commit_id") + if commit != config["environments"][branch]["commit"]: + raise ValueError("Install from the pinned Git URL (non-editable); installed commit differs or has no VCS provenance") + compiler = shutil.which(os.environ.get("CXX", "c++")) + if compiler is None: + raise RuntimeError("A C++ compiler is required, including for existing compiled augmentation kernels") + if not torch.cuda.is_available() or torch.cuda.device_count() < config["gpus"]: + raise RuntimeError( + f"Need {config['gpus']} visible CUDA GPUs; available={torch.cuda.is_available()}, " + f"count={torch.cuda.device_count()}, torch={torch.__version__}, CUDA build={torch.version.cuda}" + ) + if not verify: + for index in range(config["gpus"]): + with torch.cuda.device(index): + torch.ones(1, device=f"cuda:{index}").add_(1) + torch.cuda.synchronize() + if uses_quantized_training(config): + importlib.metadata.version("torchao") + if config.get("wandb"): + importlib.metadata.version("wandb") + gpu_memory = [torch.cuda.get_device_properties(i).total_memory for i in range(config["gpus"])] + if config.get("minimum_gpu_memory_gib") and min(gpu_memory) < config["minimum_gpu_memory_gib"] * 2**30: + raise RuntimeError("GPU memory below qualification requirement; allocate full GPUs, not MIG fractions") + record = { + "commit": commit, + "gpu_memory_bytes": gpu_memory, + "package": mini_trainer.__file__, + "python_version": platform.python_version(), + "platform": platform.platform(), + "compiler": compiler, + "cuda": torch.version.cuda, + "cpu_affinity_count": len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count(), + "torch_threads": torch.get_num_threads(), + "shared_memory_bytes": shutil.disk_usage("/dev/shm").total if Path("/dev/shm").exists() else None, + "gpus": [torch.cuda.get_device_name(i) for i in range(config["gpus"])], + "dependencies": { + d.metadata["Name"].lower(): d.version + for d in importlib.metadata.distributions() + if d.metadata["Name"].lower().replace("-", "_") != "mini_trainer" + }, + } + destination = Path(config["output"]) / f"environment-{branch}.json" + if verify: + if record != json.loads(destination.read_text()): + raise ValueError("Installed environment or GPU allocation changed since preparation") + else: + write_json(destination, record) + + +def prepare(config): + import numpy as np + import torch + + from mini_trainer.hierarchical.integration import HierarchicalBuilder + from mini_trainer.integrations.parquet import get_metadata_from_parquet + + output = Path(config["output"]) + metadata_path = qualification_parquet(config) if config.get("qualification") else config["parquet"] + progress("Building taxonomy") + taxonomy_path = config["parquet"] if config.get("full_taxonomy") else metadata_path + spec = HierarchicalBuilder.build_class_spec(dir=taxonomy_path, levels=3) + spec["resize_size"] = config["size"] + write_json(output / "class_spec.json", spec) + progress(f"Parsing image metadata; classes per level: {spec['num_classes']}") + data = get_metadata_from_parquet(metadata_path, cls2idx=spec["cls2idx"]) + if config.get("qualification"): + # The sampled Parquet is an artifact; images still live beside the source. + source_root = Path(config["parquet"]).resolve().parent + sample_root = Path(metadata_path).resolve().parent + data["path"] = [str(source_root / Path(path).relative_to(sample_root)) for path in data["path"]] + counts = Counter(data["split"]) + if not all(counts[split] for split in ("train", "validation", "test")): + raise ValueError(f"Require nonempty existing train/validation/test splits, got {counts}") + progress(f"Checking duplicates and {len(data['path']):,} image paths") + if len(set(data["path"])) != len(data["path"]): + raise ValueError("Duplicate image paths; resolve duplicates/split leakage before comparing") + checked_at = time.monotonic() + for index, (path, labels) in enumerate(zip(data["path"], data["class"], strict=True), 1): + if not Path(path).is_file(): + raise FileNotFoundError(f"Expected Parquet-adjacent images//: {path}") + if any(value is None for value in labels): + raise ValueError(f"Unmapped taxonomy: {path}") + if time.monotonic() - checked_at >= 10: + progress(f"Checked {index:,}/{len(data['path']):,} image paths") + checked_at = time.monotonic() + per_rank = config["global_batch_size"] // config["gpus"] + if counts["train"] // config["gpus"] < per_rank: + raise ValueError("Not enough training images for one global batch") + progress("Saving dataset index") + write_json(output / "data_index.json", data) + # Model construction does not need the per-image lists. + del data + smoothing = 1 / spec["num_classes"][0] + progress("Hashing source Parquet and saving dataset provenance") + write_json( + output / "dataset.json", + { + "parquet_sha256": digest(config["parquet"]), + "counts": counts, + "num_classes": spec["num_classes"], + "taxonomy_scope": "full_source" if config.get("full_taxonomy") else "indexed_rows", + "label_smoothing_per_level": [1 - (1 - smoothing) ** (1 / (i + 1)) for i in range(3)], + "image_integrity": "All indexed paths checked; image bytes are not hashed or decoded during preparation. Keep mount immutable.", + "test_used_for_training_or_selection": False, + "scope": "qualification_subset" if config.get("qualification") else "full_dataset", + "qualification": config.get("qualification"), + }, + ) + for seed in config["seeds"]: + progress(f"Building CPU FP32 starting model for seed {seed}; math threads={torch.get_num_threads()}") + progress("Pretrained download and normalized-head initialization may take time; no training is running yet") + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + model, preprocess = HierarchicalBuilder.build_model( + **spec, + model_type="efficientnet_v2_s", + device=torch.device("cpu"), + dtype=torch.float32, + hidden=True, + normalized=True, + droprate=0.1, + fine_tune=False, + ) + progress(f"Saving starting weights for seed {seed}") + torch.save(model.state_dict(), output / f"initial_seed{seed}.pt") + (output / "preprocessing.txt").write_text(repr(preprocess) + "\n") + del model + progress("Preparation worker finished") + + +def configure_figures(enabled): + """Suppress optional evaluation plots in this dedicated worker process.""" + if not enabled: + from mini_trainer.logging import MultiLogger + + MultiLogger.figures = lambda self, model: None + progress("Evaluation figures disabled; validation metrics and checkpoints remain enabled") + + +class TimedLoader: + """Measure host time waiting for next batches without synchronizing the GPU.""" + + def __init__(self, loader, synchronize=None, on_window=None): + self.loader, self.wait_seconds, self.samples = loader, 0.0, 0 + self.steps, self.windows = 0, [] + self.on_window = on_window or (lambda window: None) + self.synchronize = synchronize or (lambda: None) + + def __len__(self): + return len(self.loader) + + def __getattr__(self, key): + return getattr(self.loader, key) + + def __iter__(self): + window_start = time.monotonic() + window_samples, window_steps, window_wait = 0, 0, 0.0 + started = time.monotonic() + iterator = iter(self.loader) + while True: + try: + batch = next(iterator) + except StopIteration: + self.wait_seconds += time.monotonic() - started + return + self.wait_seconds += time.monotonic() - started + self.samples += len(batch[0]) + self.steps += 1 + yield batch + if self.steps % 32 == 0 or self.steps == len(self.loader): + self.synchronize() + now = time.monotonic() + self.windows.append( + { + "steps": self.steps - window_steps, + "samples": self.samples - window_samples, + "seconds": now - window_start, + "loader_wait_seconds": self.wait_seconds - window_wait, + } + ) + self.on_window(self.windows[-1]) + window_start, window_samples = now, self.samples + window_steps, window_wait = self.steps, self.wait_seconds + started = time.monotonic() + + +def instrument(output, config=None): + """Record synchronized phase times per rank without changing optimizer logic.""" + import torch + + import mini_trainer.trainer as trainer + from mini_trainer.logging import MultiLogger + + # The normal logger resets peak stats within a phase. Disable only that + # measurement reset so phase-level CUDA peaks include all steps. + MultiLogger._reset_cuda_memory_stats = staticmethod(lambda: None) + rank = int(os.environ.get("RANK", "0")) + + def component_timer(function, component): + @functools.wraps(function) + def measured(*args, **kwargs): + started = time.monotonic() + try: + return function(*args, **kwargs) + finally: + with (output / f"components-rank{rank}.jsonl").open("a") as handle: + handle.write(json.dumps({"component": component, "seconds": time.monotonic() - started}) + "\n") + + return measured + + MultiLogger.figures = component_timer(MultiLogger.figures, "figures") + trainer.save_on_master = component_timer(trainer.save_on_master, "checkpoint") + + def wrap(function, phase): + @functools.wraps(function) + def measured(*args, **kwargs): + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + started = time.monotonic() + bound = inspect.signature(function).bind(*args, **kwargs) + + def save_window(window): + with (output / f"windows-rank{rank}.jsonl").open("a") as handle: + handle.write(json.dumps({"phase": phase, "epoch": bound.arguments["epoch"], **window}) + "\n") + + loader = TimedLoader(bound.arguments["data_loader"], torch.cuda.synchronize, save_window) + bound.arguments["data_loader"] = loader + value = function(*bound.args, **bound.kwargs) + torch.cuda.synchronize() + record = { + "phase": phase, + "epoch": bound.arguments["epoch"], + "samples": loader.samples, + "steps": loader.steps, + "windows": loader.windows, + "loader_wait_seconds": loader.wait_seconds, + "seconds": time.monotonic() - started, + "max_memory_allocated": torch.cuda.max_memory_allocated(), + "max_memory_reserved": torch.cuda.max_memory_reserved(), + } + if phase == "validation": + record["selection_metric"] = float(value) + with (output / f"phases-rank{rank}.jsonl").open("a") as handle: + handle.write(json.dumps(record) + "\n") + if config and config.get("wandb"): + import torch.distributed as dist + import wandb + + totals = torch.tensor( + [record["seconds"], record["max_memory_allocated"], record["max_memory_reserved"], loader.wait_seconds], + device="cuda", + dtype=torch.float64, + ) + if dist.is_initialized(): + dist.all_reduce(totals, op=dist.ReduceOp.MAX) + if rank == 0: + seconds, allocated, reserved, wait = totals.tolist() + wandb.log( + { + f"qualification/{phase}_images_per_second": loader.samples * config["gpus"] / seconds, + f"qualification/{phase}_seconds": seconds, + f"qualification/{phase}_peak_allocated_bytes": allocated, + f"qualification/{phase}_peak_reserved_bytes": reserved, + f"qualification/{phase}_loader_wait_seconds": wait, + "qualification/epoch": bound.arguments["epoch"], + } + ) + write_json(output / "wandb-run.json", {"id": wandb.run.id, "url": wandb.run.url}) + return value + + return measured + + trainer.train_one_epoch = wrap(trainer.train_one_epoch, "train") + trainer.evaluate = wrap(trainer.evaluate, "validation") + + +def configure_wandb(config, name): + if not config.get("wandb"): + return {} + # Authentication remains the existing W&B integration's responsibility. + # A trial-specific shared ID keeps eight ranks together and trials separate. + import hashlib + + from mini_trainer.logging import MetricLogger, WandbLogger + + run_id = hashlib.sha256(f"{config['output']}/{name}".encode()).hexdigest()[:16] + return { + "logger_cls": [MetricLogger, WandbLogger], + "logger_cls_extra_kwargs": [ + {}, + { + "project": os.environ.get("WANDB_PROJECT", "mini-trainer-ddp"), + "run_name": f"{Path(config['output']).name}-{name}", + "run_id": run_id, + }, + ], + } + + +def configure_training_checks(config, directory): + """Check restored state before DDP wrapping/compilation and optimizer updates.""" + import torch + + import mini_trainer.train as train_module + + original = train_module.train + + def equal(actual, expected): + if isinstance(expected, torch.Tensor): + assert actual.dtype == expected.dtype and torch.equal(actual.detach().cpu(), expected) + elif isinstance(expected, dict): + assert actual.keys() == expected.keys() + for key in expected: + equal(actual[key], expected[key]) + elif isinstance(expected, (list, tuple)): + assert len(actual) == len(expected) + for a, b in zip(actual, expected, strict=True): + equal(a, b) + else: + assert actual == expected + + def checked(**kwargs): + kwargs["weight_store_rate"] = 1 + if config.get("checkpoint"): + saved = torch.load(config["checkpoint"], map_location="cpu", weights_only=False) + assert kwargs["start_epoch"] == saved["epoch"] + 1 == config["resume_epoch"] + for key in ("model", "optimizer", "lr_scheduler", "scaler"): + if key == "scaler" and kwargs[key] is None: + continue + equal(kwargs[key].state_dict(), saved[key]) + write_json( + directory / f"restore-rank{os.environ.get('RANK', '0')}.json", + { + "checkpoint_sha256": digest(config["checkpoint"]), + "start_epoch": kwargs["start_epoch"], + "model_optimizer_scheduler_scaler_restored": True, + }, + ) + del saved + return original(**kwargs) + + train_module.train = checked + + +def train(config, name): + import torch + + from mini_trainer.hierarchical.integration import HierarchicalBuilder + from mini_trainer.train import main + from mini_trainer.training import MuonAuxAdamW + + run = next(run for run in plan(config) if run["name"] == name) + preflight(config, run["branch"], verify=True) + root = Path(config["output"]) + directory = root / "runs" / name + # Isolate compiler caches by run and rank; compilation cost is included. + rank = os.environ.get("RANK", "0") + cache_root = os.environ.get("MT_COMPILER_CACHE_ROOT", "/tmp") + if any(char.isspace() for char in cache_root): + raise ValueError("MT_COMPILER_CACHE_ROOT must not contain whitespace (compiler toolchain limitation)") + cache = Path(tempfile.mkdtemp(prefix="mt-compile-", dir=cache_root)) + os.environ["TORCHINDUCTOR_CACHE_DIR"] = str(cache / "inductor") + os.environ["TRITON_CACHE_DIR"] = str(cache / "triton") + directory.mkdir(parents=True, exist_ok=True) + write_json(directory / f"compiler-cache-rank{rank}.json", {"path": str(cache)}) + if not torch.cuda.is_available() or torch.cuda.device_count() < config["gpus"]: + raise RuntimeError("Allocated GPUs no longer match configuration") + configure_figures(config.get("figures", True)) + instrument(directory, config) + options = run["options"].copy() + logging_options = configure_wandb(config, name) + if config.get("checkpoint") or config.get("mode") == "scaling": + configure_training_checks(config, directory) + loader = { + "batch_size": config["global_batch_size"] // config["gpus"], + "num_workers": config["num_workers_per_rank"], + "data_index": str(root / "data_index.json"), + "cache": None, + "splits": ("train", "val"), + "resample": False, + } + if "cuda_prefetch" in options: + loader["cuda_prefetch"] = options.pop("cuda_prefetch") + main( + input=config["parquet"], + output=str(directory), + name="model", + class_spec=str(root / "class_spec.json"), + checkpoint=config.get("checkpoint"), + builder=HierarchicalBuilder, + epochs=config["epochs"], + size=config["size"], + seed=run["seed"], + device="cuda", + dtype="float16", + ema=False, + model_builder_kwargs={ + "model_type": "efficientnet_v2_s", + "weights": str(root / f"initial_seed{run['seed']}.pt"), + "model_args": {"pretrained": False}, + "hidden": True, + "normalized": True, + "droprate": 0.1, + "fine_tune": False, + }, + dataloader_builder_kwargs=loader, + optimizer_builder_kwargs={"optimizer_cls": MuonAuxAdamW, "lr": 0.001, "weight_decay": 0.01}, + criterion_builder_kwargs={"weighted": True, "label_smoothing": None, "weights": [1, 1, 1]}, + regularizer_builder_kwargs={"strength": 0.1}, + lr_schedule_builder_kwargs={"warmup_epochs": 0.25}, + # Predeclare hierarchical columns: otherwise the logger first writes a + # flat CSV header, then rejects additional columns after validation. + logger_builder_kwargs={ + **logging_options, + "verbose": True, + "statistics": [ + "acc1", + "acc5", + "loss", + "lr", + "item/s", + "mem", + "step", + "time", + "eta", + "epoch", + "type", + "acc1/lvl1", + "acc5/lvl1", + "acc1/lvl2", + "acc5/lvl2", + "loss/lvl0", + "loss/lvl1", + "loss/lvl2", + ], + }, + **options, + ) + if config.get("wandb"): + import wandb + + wandb.finish() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("preflight", "prepare", "train")) + parser.add_argument("config", type=Path) + parser.add_argument("--branch", choices=("master", "quant")) + parser.add_argument("--run") + args = parser.parse_args() + config = json.loads(args.config.read_text()) + if args.action == "preflight": + preflight(config, args.branch) + elif args.action == "prepare": + prepare(config) + else: + train(config, args.run) + + +if __name__ == "__main__": + main() diff --git a/dev/worktrees.md b/dev/worktrees.md new file mode 100644 index 00000000..b8c56ebe --- /dev/null +++ b/dev/worktrees.md @@ -0,0 +1,78 @@ +# Parallel local development + +Use one Git worktree per active feature. Worktrees have independent tracked files, +indexes and checked-out branches; commits, refs and Git configuration are shared. +The current prototype exploration starts from `quant`, whose diagnostic functions +are the reference for that work. Choose the base explicitly for other features. + +From the main checkout: + +```bash +git worktree list +git worktree add -b feature/my-feature .worktrees/my-feature quant +``` + +`/.worktrees/` is ignored on branches carrying this guide. For older branches, +add `/.worktrees/` once to the main checkout's `.git/info/exclude` (local only, +shared by worktrees). Do not copy uncommitted edits into another branch unless +they belong to its task. Existing main-checkout changes stay in place. + +## Reuse dependencies without changing them + +When both branches use the same dependencies, a worktree can reuse the existing +environment through a `.venv` symlink. This shares installed packages; it does +not provide independent environments. No branch using that environment should +install, remove or synchronize packages while others are using it. + +From the main checkout, before entering the new worktree: + +```bash +ln -s "$PWD/.venv" .worktrees/my-feature/.venv +cd .worktrees/my-feature +export PYTHONPATH="$PWD" +CUDA_VISIBLE_DEVICES='' .venv/bin/python -c 'import mini_trainer; print(mini_trainer.__file__)' +bash dev/check.sh static +bash dev/check.sh test tests/utils/test_dendrogram.py +``` + +The import check must point into the new worktree. Set `PYTHONPATH` separately +in each worktree's shell: it also gives Python subprocesses the correct source +root. Prefer `.venv/bin/python -m ...` over installed console scripts, which can +otherwise resolve an editable installation in the original checkout. + +Do not run `uv sync`, `uv pip install` or ordinary `uv run` against a shared +environment. Use its executables directly, or `uv run --no-sync`. The shared +environment remains subject to the README's explicit PyTorch-backend rules. + +If a branch needs different dependencies, first unlink only its `.venv` symlink +(`test -L .venv && unlink .venv`), then create a separate environment using the +README's explicit installation command and intended CPU/CUDA backend. Never +delete the symlink target. Separate environments need their own package/import +verification; CPU validation does not establish GPU correctness. + +## Keep experiments isolated + +- Use a separate terminal/session per worktree and check `pwd` and + `git branch --show-current` before edits, checks or commits. +- Keep generated reports in that worktree's ignored `tmp/`. Pass existing large + checkpoints and datasets by absolute path rather than copying them into Git. +- Do not run multiple experiments against the same output directory or change a + shared dependency environment concurrently. GPU, RAM and disk resources remain + shared even though the source trees are separate. +- Stage explicit paths and inspect the staged diff. A worktree does not protect + another worktree from operations on shared branch refs or repository config. +- Commit focused work before merging or cherry-picking it. Do not switch the + main checkout away from ongoing work just to inspect a feature branch. + +When a feature is integrated and its worktree has no changes or needed outputs: + +```bash +git worktree list +git -C .worktrees/my-feature status --short +git worktree remove .worktrees/my-feature +``` + +Inspect and retain ignored reports before removal. Do not use `--force` to +discard work. Branch deletion is a separate decision after integration. + +Agents must also follow the [worktree coordination rules](../.agents/rules/worktrees.md). diff --git a/docs/archive/benchmark-history.md b/docs/archive/benchmark-history.md new file mode 100644 index 00000000..188a9c18 --- /dev/null +++ b/docs/archive/benchmark-history.md @@ -0,0 +1,11 @@ +# Historical benchmark notebook + +The detailed chronological notebook is preserved in [Git at `f5c69e7`](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md). +It includes exact local configurations, negative experiments and superseded results. +Use [current findings](../benchmarks.md) for conclusions and +[the roadmap](../quantization-roadmap.md) for remaining work. + +Raw evidence and final model retention are described in +[the artifact inventory](../quantization-artifacts.md). Old temporary paths require +restoration; historical commands use the former flat benchmark module paths. +Current commands live in [the benchmark guide](../../dev/benchmarks/README.md). diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 1a88ee2a..31ea8867 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,74 +1,68 @@ -# Continuous dataset benchmarks +# Benchmark findings -[Browse benchmark runs and their summaries](https://github.com/asgersvenning/mini_trainer/actions/workflows/benchmarks.yml). -The workflow is defined in this repository; results start appearing there once the -change is merged and runs execute. Initial local evidence is recorded below. +These are the current conclusions of the quantization branch, based on local +measurements. They do not certify HPC, Spark/desktop, or ARM performance. +The [branch roadmap](quantization-roadmap.md) defines the remaining work and +completion criteria. Exact configurations, trials and superseded findings remain +in the [historical experiment record](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md). -The benchmark foundation exercises the actual training, checkpoint reload and -inference paths. CPU is a fast verification milestone, not the scope boundary. -The progression is synthetic oracle → MNIST → hierarchical Blair, with Birds and -iNaturalist 2021 reserved for larger, more expensive follow-ups. +## Representative workload -Each run produces a visible Actions summary and a downloadable artifact containing -JSON reports, predictions, dataset/split manifests, configurations, checkpoints and -logs. Failures are retained alongside successful results. The artifact excludes -synthetic image files, which can be regenerated from the recorded seed; source -images for real datasets are never uploaded by the workflow. +EfficientNetV2-S, symmetric hidden layer and normalized flat or hierarchical +classifier; reviewed Blair splits and paired seeds. Full training and +parameter-frozen fine-tuning are different workloads. Synthetic 10k/100k-class +heads measure capacity, not classification quality. Full million-class runs are +reserved for larger machines. -Reports identify the source revision and source-content hash, lockfile hash, -installed versions, dataset/checkpoint hashes, seeds, class mappings, device, -precision, cache settings and explicitly exercised capabilities. Results marked -`completed` only establish that the real-data pipeline ran: they do not claim a -quality improvement. Synthetic profiles have an exact 100% oracle quality gate. +## Evidence that drives the next decisions -Artifacts are configured for 90-day retention. This is an initial reporting surface, -not a permanent model zoo. A durable results index/dashboard should archive these -versioned records before expiry, retain failures, and compare matching configurations -across revisions. It must separate quality from speed and avoid combining CPU/GPU, -precision, dataset, or dependency changes into a misleading trend. - -## Initial local evidence - -These are baseline observations, not proposed model-quality or speed improvements. -They use seed 42, one CPU thread, zero loader workers, no augmentation/EMA, and small -offline models. The synthetic runs use 12 epochs; real-data runs use 5 epochs. - -| Profile | Held-out accuracy | Scope | +| Regime | Measured finding | Consequence | | --- | --- | --- | -| Synthetic CPU float32 | 100% | Exact repeatability verified within the local environment | -| Synthetic CUDA float32 | 100% | Strict deterministic algorithms; CUDA cache | -| Synthetic CUDA float16 | 100% | AMP and CUDA cache | -| Synthetic CUDA bfloat16 | 100% | AMP and CUDA cache | -| MNIST CUDA float16 | 97.12% | AMP, CUDA cache, explicit nondeterministic profile | -| MNIST CPU float32 | 97.3% | 4,000 train / 1,000 validation / 5,000 test | -| Blair CUDA float16 | 64.3% species / 80.4% parent | 3,704 train / 912 validation / 1,161 test | +| Native INT8, full pretrained Blair training | About 3.4% lower allocated peak memory; timing tied or worse in short three-seed studies. Five-epoch hierarchical parent F1 fell in all three seeds, up to 5.10 points; precision up to 5.38 points. | The short full-training recipe is not a demonstrated useful trade-off. Profile backbone costs and qualify time to comparable quality. | +| Native INT8, parameter-frozen Blair fine-tuning | About 17.8% lower allocated peak; mixed timings. Five-epoch absolute quality was below full training. | Qualify fine-tuning separately; memory savings do not establish equal-quality training efficiency. | +| Longer hierarchical training | One 20-epoch pair: leaf/parent F1 differences +0.331/−0.082 points, but INT8 training took 9.2% longer locally. | Longer budgets can change the quality result. Repeat seeds and measure time to useful quality on target hardware. | +| BatchNorm refresh | Training-only refresh improved some early checkpoints; twelve-checkpoint qualification had mixed held-out effects and remaining parent-level regressions. | No automatic refresh is recommended. Running-statistic sensitivity is a diagnostic finding, not a demonstrated universal fix. | +| 100k-class native training capacity | Bounded normalization backward reduced full-training peak allocation by 17.4%; corrected frozen comparison saved 6.6%. Compiler/graph variants had mixed speed and reserved-memory results. | Large heads can benefit, but distinguish allocated, reserved, setup and steady-state memory. These are synthetic capacity results. | +| ONNX CPU inference on x86 | Unsigned activation/per-channel INT8 recipe executed 170 integer convolutions and two head GEMMs; three trials per head showed 44–54% lower warm batch-one latency and 45–48% lower RSS. Largest observed metric loss was 2.956 points in hierarchical leaf precision. | Promising edge candidate; verify kernels, quality and sustained resources on ARM before recommending it there. | +| TensorRT inference with 100k random classes | INT8 engine about 49% smaller. Batch 1/8 was slower; batch 64 was effectively tied. Batch-64 device snapshots were 140 MiB lower: 37% of the post-initialization increment, 9% of total warm usage. | A local memory-saving candidate, not a demonstrated speedup or trained large-vocabulary quality result. | +| Native QT export | Integer-forward ONNX is supported in a limited tested runtime path, but CUDA MatMulInteger falls back to CPU and TensorRT rejects the native representation. Explicit floating materialization followed by static calibration provides a separate deployable route. | Do not claim the calibrated export preserves the native dynamic quantizer. Validate its quality and placement independently. | +| Large-head float export parity | Some near-zero scores required an explicit 1e-4 absolute tolerance; FP64 diagnostics were consistent with accumulated rounding. | Keep the public default parity gate. Qualify any profile-specific tolerance with numerical evidence. | -GPU observations use an NVIDIA GeForce RTX 3080 Ti Laptop GPU. The environment -uses Python 3.13.7 and PyTorch 2.12.0. Single-run timing is diagnostic; it is not a -portable performance gate. CPU synthetic training-call wall time was about 2.2 s, -MNIST CPU about 11 s, and Blair GPU about 13.4 s. These scopes include setup, -training, validation, logging and checkpoints, but exclude dataset inventory and -final held-out inference. Runs were not a controlled throughput comparison. +## Measurement rules -Strict deterministic mode rejected CUDA adaptive-pooling backward in the real-data -backbone. The explicit `--allow-nondeterministic` profile permits those operations -and records that choice. Seeds and preserved inputs make the experiment repeatable; -they do not promise bitwise-identical GPU results. Neither AMP nor CUDA caching is -inference quantization or quantization-aware training. +- Compare practical floating baselines: BF16/FP16 training where supported, + FP16 TensorRT and floating ONNX CPU. Use matched data, preprocessing and budgets. +- Evaluate Macro-F1, Macro-Recall, Macro-Precision, Coverage and Theil's U with + `mini_metrics`, at leaf and parent levels. Retain undefined values and negative + results. Differences multiplied by 100 include Theil's U; label that convention. +- Accept a few points of quality loss only alongside a substantial measured + speed/cost or memory benefit in the intended workload. +- Separate cold setup, first execution, warm latency, end-to-end throughput, + loading/transfers, allocated/reserved GPU memory and process/device snapshots. + Device-wide snapshots are not transient peaks or per-process allocation. +- Runs overlapping CPU checks do not support timing/resource claims. Random + large-class heads and synthetic ARM metadata do not establish real quality or + ARM execution. +- Historical epoch CSVs generated while all statistic loggers were disabled are + invalid convergence evidence; their independently evaluated `last.pt` results + remain usable. `best.pt` selection from those runs is not trustworthy. -## Coverage still needed +## Reproduction and visibility -The reports currently mark EMA, distributed training, augmentation, quantization -and ONNX integration as unexercised by this benchmark suite. Existing focused tests -provide other evidence, but do not make those boxes true for these dataset runs. -Add explicit profiles and comparison criteria before claiming coverage or improvement. -The known EMA continuation failure remains recorded in the roadmap. +Use the [benchmark command index](../dev/benchmarks/README.md): +[training](../dev/benchmarks/training.md), +[inference](../dev/benchmarks/inference.md), and +[reporting](../dev/benchmarks/reporting.md). +CPU and TensorRT composed reports bind quality to retained runtime evidence; +compact history distinguishes CPU medians/RSS from GPU paired timing/snapshots. +New CPU reports also retain and verify the requested trial budget. -The plain AdamW/SGD step-tracking blocker discovered by this baseline is now fixed. -The trainer gates scheduler/EMA advancement on completed optimizer steps and the -native fused-optimizer AMP overflow signal. Real CPU and CUDA overflow regressions -cover MuonAuxAdamW, AdamW, SGD and fused AdamW/SGD. This establishes compatibility; -it does not yet measure their relative model quality. See the [step contract](../dev/README.md#optimizer-step-contract). +An opt-in default-branch publisher connects compact TensorRT artifacts to monthly +draft-release storage and GitHub Pages. Live activation, authenticated storage, +public rendering and target-runner execution remain unverified. CPU producer and +training-history handoffs are follow-up integration work, not established services. -See the [benchmark guide](../dev/benchmarks/README.md) for local commands, GPU runner -configuration, real-data inputs and reproduction details. +Raw temporary paths cited by older notes are now governed by the +[artifact-retention record](quantization-artifacts.md). Downloaded datasets, +original research files and the working environment are separate from disposable +benchmark outputs. diff --git a/docs/onnx.md b/docs/onnx.md index 08616c06..7e2034b3 100644 --- a/docs/onnx.md +++ b/docs/onnx.md @@ -38,7 +38,7 @@ other score conversion is added: output semantics are those of `model.eval()`. Classifier metadata includes the effective class mappings after masking. In particular, hierarchical outputs remain separate tensors in their original order. -A private CPU copy uses the example tensor's dtype. The source model's weights, +A private copy uses the example tensor's dtype and `reference_device` (CPU by default). The source model's weights, training modes, device and classifier caches remain untouched. Root DataParallel, DDP and compiled wrappers are unwrapped. Use float32 examples for portable CPU verification, including when the source was trained in bfloat16. Float16/float64 @@ -98,7 +98,120 @@ used by BioCLIP, plus every classifier head family. This is representative backe coverage, not certification of every model in the catalog. Arbitrary custom operators and data-dependent Python control flow remain subject to the [PyTorch ONNX exporter's support](https://docs.pytorch.org/docs/stable/onnx_export.html). -GPU providers, quantized graphs and arbitrary spatial dimensions are not validated. +The actual EfficientNetV2-S backbone with symmetric normalized flat/hierarchical +heads is also covered by offline dynamic-batch export tests. Trained Blair +checkpoints passed ONNX Runtime CPU parity on real images; see the +[deployment experiment](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md#efficientnetv2-onnx-cpu-export-and-inference-quantization). +Local [CUDA placement checks](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md#onnx-cuda-provider-placement) expose +CPU fallback for native integer heads and floating execution for calibrated +convolutions. Target GPU hardware, ARM execution and arbitrary spatial dimensions +remain unvalidated. +An initial signed MinMax INT8 recipe lost substantial accuracy and retained +floating convolutions. A follow-up unsigned Percentile recipe executed all +convolutions as QLinearConv and roughly halved warm local CPU inference latency, +with remaining quality losses measured through `mini_metrics`; see the +[calibration and metric results](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md#onnx-activation-calibration-execution-coverage-and-macro-metrics). +It remains exploratory, with no agreed production quality gate or target-device +verification. Native CUDA QT checkpoint export is a separate path described below; +these floating-checkpoint PTQ experiments do not validate it. These local bundles are a foundation for Hugging Face hosting. Model cards, evaluation attachments and Hub upload commands remain separate roadmap work. + + +## Native INT8 training checkpoints + +Native `cuda-int8-linear` checkpoints can export their captured integer forward +with an explicit CUDA reference. This requires the existing quantization and +export extras in a compatible CUDA environment; deployment itself needs only +ONNX Runtime, NumPy and external preprocessing. + +```bash +CUDA_VISIBLE_DEVICES=0 .venv/bin/mt_export --weights native-int8-weights.pt \ + --output native-int8-onnx --reference-device cuda:0 +``` + +The Python API uses `export_onnx(model, float32_images, destination, +reference_device="cuda:0")`. Float32 inputs and a CUDA reference are required for +this backend. The CLI uses the scoped native-weight loader, retaining restricted +checkpoint loading. The export copy is frozen to let the exporter unpack integer +parameter storage; the caller's weights and gradient flags are preserved. + +The graph retains the training backend's dynamic symmetric row quantization, +including clipping, ties-to-even rounding and zero-row behavior. Its scaled INT8 +products lower to `MatMulInteger` with bounded INT32 partial accumulation and +floating row/column scales. Long contractions combine partial sums in INT64 +before converting and applying scales, preventing accumulator saturation. Activation codes are represented as unsigned codes with zero point 128; +this preserves the signed values exactly. Weights stay signed INT8. No calibration +set, floating-weight substitution or replacement classifier is used. Convolutions +remain floating, as they do in native QT training. This is distinct from the +static Percentile recipe that also quantizes convolutions. + +CUDA reference execution temporarily disables CUDA autocast and TF32 in cuDNN +and floating matrix products, restoring the caller settings afterward. Real trained images exposed a parity +failure with TF32 enabled; full-FP32 reference execution passed the original +rtol=1e-4/atol=1e-5 checks. The manifest records the reference device and TF32 choice. +This does not promise matching scores against AMP or TF32 evaluation of the same +checkpoint, whose rounding can change the subsequent integer activation codes. + +Tests cover normalized symmetric flat/hierarchical heads, EfficientNetV2-S, +active-class filtering, dynamic batches, checkpoint CLI loading and numerical +edge cases. Trained Blair checkpoints also passed checks on eight real validation +images at batches 1, 2, 4 and 8. An exported graph still needs runtime profiling +and quality evaluation on the intended provider. Local CUDA-provider execution +retains CPU MatMulInteger operations; it does not establish integer GPU execution. +Target GPU/ARM deployment, million-class export capacity and production performance +of this native path remain unverified. On the full Blair validation split, +top-1 predictions and the +requested macro metrics matched the full-FP32 CUDA reference, but some image +scores exceeded the strict export tolerance; see the +[full-validation results](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md#native-onnx-full-validation-quality-and-numerical-limits). +Supply representative `verification_inputs` and evaluate deployment thresholds +separately; passing the default sample checks does not establish universal score +parity or confidence-threshold equivalence. The generic exporter does not +impose a model allowlist; configurations outside this tested coverage must pass +the same export and parity checks before a bundle is published. + +## Explicit materialization for deployment calibration + +`mt_export --materialize-int8-training` exports a **separate floating model** from +a native INT8 training checkpoint. This is an opt-in route to subsequent static +calibration for runtimes that cannot execute the native dynamic quantizer. The +ordinary export path above remains unchanged. + +```bash +mt_export --weights native-int8-last.pt --output materialized-onnx \ + --input-shape 3 128 128 --materialize-int8-training +``` + +The conversion checks the recorded native recipes, copies the state, materializes +INT8 weight representations, and removes the recipe that would restore native +training tensor types. It preserves class metadata, active-class masks and other +buffers. Normalized directions use signed integer codes, with magnitudes set to +zero for zero-scale rows. This follows the native effective-weight formula +and handles zero scales without introducing an ordinary weight-normalization +divide-by-zero. Compatible parameter ties are retained by normal model loading; +incompatible tied roles/views fail instead of silently loading different values +into one shared parameter. Undefined zero-code directions and invalid/nonfinite states fail. +The source checkpoint is never rewritten, and neither optimizers nor training +state are carried into this deployment artifact. + +**Dynamic activation quantization is removed.** ONNX verification compares against +the materialized floating model, not against the native training forward. Its +manifest records the original checkpoint hash, the floating state hash and an +explicit `source.quantized_training_materialization` recipe, including +`dynamic_activation_quantization_preserved=false` and +`training_resume_supported=false`. The original checkpoint remains the source +for native training/resume. Materialization holds floating weights in memory; +it is not a training-memory optimization. + +Use the maintained [input preparation](../dev/benchmarks/inference.md#maintained-image-input-preparation) +and [calibration](../dev/benchmarks/inference.md#maintained-onnx-calibration-command) +commands on this artifact. For the tested TensorRT recipe, choose signed symmetric +activations, signed per-channel weights and floating biases, with training-only +calibration data. Build and inspect a new engine for its destination device. +Compare native predictions, materialized predictions and the calibrated deployment +candidate on the same held-out samples using all five requested mini_metrics +metrics. Use a practical FP16 baseline for efficiency comparisons; successful +materialization/export alone does not establish acceptable quality, integer GPU +execution or a worthwhile cost reduction. diff --git a/docs/quantization-artifacts.md b/docs/quantization-artifacts.md new file mode 100644 index 00000000..5c145f90 --- /dev/null +++ b/docs/quantization-artifacts.md @@ -0,0 +1,58 @@ +# Quantization artifact retention + +The repository's historical benchmark notes describe experiments, not a guarantee +that every temporary binary remains in the checkout. Cleanup consolidates local +evidence under ignored `local-evidence/quantization-2026-09-09/`; no model, dataset +or generated result is committed by this operation. + +Historical cleanup counts, disk-space observations and the test snapshot are +preserved in the [dated agent handoff](../.agents/notes/2026-09-09-quantization-cleanup.md). +They do not establish current artifact availability or test status. This guide +covers the retained layout and restore procedure. + +## Retained locally + +- `reports-inputs.tar.gz`: reports, metrics/predictions, prepared NPZ inputs, + experiment scripts/configurations, logs, traces and small synthetic image files. + Original `tmp-*` relative paths are preserved inside the archive. +- `retained/`: final `last.pt` models for the flat/hierarchical three-seed studies + and the longer hierarchical pair; floating/materialized and calibrated ONNX + deployment bundles for both representative heads. Their original relative + directory structure is preserved. +- `inventory.json`: archived/retained paths, sizes and SHA-256 digests, plus the + discarded-file inventory and runtime-directory totals. Archive content and + retained models are verified before their temporary source directories are removed. +- `README.md`: local cleanup totals and restore instructions. + +Prepare optional GPU runtimes explicitly on the target using the +[inference guide](../dev/benchmarks/inference.md); the archive does not include a +portable working environment. + +## Removed as disposable + +Intermediate and duplicate checkpoint/model exports, synthetic large-head binary +models, laptop-specific TensorRT engines, temporary package installations, and +completed pytest outputs. Final optimizer-resume snapshots are not retained in +this compact handoff: exact continuation from discarded experiments requires +regenerating the run. Retained final model weights support inference/quality +re-evaluation, not a claim of complete optimizer/RNG restoration. + +## Restore selected evidence + +Run from the repository root. List the archive before choosing paths: + +```bash +tar -tzf local-evidence/quantization-2026-09-09/reports-inputs.tar.gz +tar -xzf local-evidence/quantization-2026-09-09/reports-inputs.tar.gz \ + tmp-cpu-deployment-flat tmp-cpu-deployment-hierarchical +``` + +This restores reports and their child evidence for archival/review without engines +or model weights. To rerun inference, restore the corresponding inputs and copy +the selected bundle from `retained/` to the intended location. Rebuild TensorRT +engines on the target. Original reports keep original provenance paths and hashes; +do not rewrite them to make discarded binaries appear available. + +The archive is a local handoff, not a remote backup or a live result-hosting service. +Transfer it deliberately with the required retained models when target access is +available. Public compact history contains only the selected reporting projection. diff --git a/docs/quantization-roadmap.md b/docs/quantization-roadmap.md new file mode 100644 index 00000000..fa854d9f --- /dev/null +++ b/docs/quantization-roadmap.md @@ -0,0 +1,128 @@ +# Quantization feature-branch roadmap + +This is the current plan for `quant`. The local implementation and evidence are +substantial, but the goal remains incomplete until useful trade-offs are verified +on the intended hardware. After repository consolidation, pause further broad +laptop sweeps and reporting expansion. Resume with a specific bottleneck, +target run or integration requirement from the sequence below. + +## Scope and acceptance + +Use EfficientNetV2-S with symmetric hidden layers and normalized flat and +hierarchical classifiers. Evaluate full training and parameter-frozen fine-tuning +separately. Use reproducible synthetic/oracle checks, MNIST integration and reviewed +Blair splits; use 10k/100k-class synthetic heads for capacity. Larger class counts +belong on appropriately provisioned hardware. Synthetic capacity does not prove +large-vocabulary classification quality. + +| Target | Required execution | Evidence needed to finish | +| --- | --- | --- | +| HPC: A40/A100/B300-class systems with AMD EPYC | PyTorch training | Paired convergence/time to comparable quality, throughput, cold/setup costs, host and allocated/reserved device memory, actual loading/transfer costs, checkpoint/resume correctness. Record each tested GPU/runtime/allocation. | +| Local: NVIDIA Spark or intended RTX desktop/Ryzen system | PyTorch training/fine-tuning and ONNX GPU inference | Full/frozen training evidence plus target-built inference engines, actual integer placement, paired quality, realistic batches, end-to-end inference cost and memory. Verify exact installed device and architecture first. | +| Edge: Raspberry Pi or comparable ARM device | ONNX CPU inference | Installed runtime/operator compatibility, five-metric quality, batch-one latency, sustained throughput, process memory, preprocessing and deployment packaging under realistic thread/power/thermal conditions. | + +The quality contract is Macro-F1, Macro-Recall, Macro-Precision, Coverage and +Theil's U through `mini_metrics`, including parent levels. A few points of loss +can be acceptable with a substantial measured speed/cost or memory benefit. +Choose and record per-profile quality/resource gates before qualification; do not +hide a failed metric behind accuracy or engine size. Keep baselines practical: +BF16/FP16 training where supported, FP16 TensorRT, and floating ONNX CPU. + +## 1. Finish local preparation and handoff + +The implementation, current findings, command guides, grouped tests and retained +evidence should form a reviewable branch milestone. This stage does not require +new target hardware or another performance sweep. + +- Run static checks and the complete grouped suite; preserve test collection, + checkpoint fixture imports and spawn behavior. Keep optional/GPU skips explicit. +- Retain essential trained final models, deployable ONNX sources, predictions, + reports, input identities and scripts; remove disposable duplicate/intermediate + models and laptop-specific engines. See [artifact retention](quantization-artifacts.md). +- Use existing [training](../dev/benchmarks/training.md), + [inference](../dev/benchmarks/inference.md) and + [reporting](../dev/benchmarks/reporting.md) commands as the handoff. Avoid a + second implementation of model construction, evaluation or configuration. +- Keep the branch unmerged until reviewed. Repository settings and public + publication require a deliberate activation step after review. + +Exit: clean committed branch, unchanged collected test cases, passing checks, +concise documentation, and an inventory of retained/restorable evidence. + +## 2. Obtain target runs and establish practical baselines + +**Dependency:** access to the target machines, or an operator who can run the +commands and return the full artifacts. No local measurement can remove this +dependency. Record GPU/CPU identity, architecture, memory, OS, driver/runtime, +PyTorch/ONNX versions, thread/worker limits and allocation conditions. + +1. Prepare explicitly selected backend environments; do not synchronize away a + working CUDA environment. Verify TorchAO/kernel support on the actual GPU and + host architecture. Keep `mini_metrics` in an explicit compatible interpreter. +2. Stage the reviewed dataset/split/taxonomy and training-only calibration inputs. + Keep preprocessing and input/model hashes identical across each pair. +3. Run a bounded correctness pilot: model construction, a training step, checkpoint + reload, inference and operator placement. Reject unsupported execution before + launching a long benchmark. +4. Run `bash dev/check-benchmarks.sh qt-efficientnet FRESH_RESULTS` with the documented + data/interpreter variables. Its default covers both heads, full/frozen training, + BF16/INT8 and three seeds. Select longer budgets explicitly where needed; + five-epoch quality is not a settled convergence comparison. +5. Rebuild TensorRT engines on the target with + `bash dev/check-tensorrt-deployment.sh FRESH_RESULTS`. On ARM, use + `python -m dev.benchmarks.inference.cpu_deployment` with the reviewed baseline/candidate, + manifest and representative inputs. Begin with explicit conservative threads. +6. Collect fresh-process repeated runs without competing benchmark jobs. Include + cold startup, warm execution and end-to-end input handling as separate scopes. + Return raw reports and artifacts even when a stage fails. + +Exit: reproducible floating and quantized baseline evidence for the actual target; +an identified dominant cost or a demonstrated useful trade-off. Scope claims to +tested configurations rather than extrapolating across hardware families. + +## 3. Unlock measured performance bottlenecks + +The [findings](benchmarks.md) summarize the evidence behind these priorities. +Choose the next row using target profiles; do not implement all possible precision +formats or distributed backends merely because they are available. + +| Observed bottleneck | Action that can unlock it | Verification gate | +| --- | --- | --- | +| Full EfficientNet training gets only about 3.4% peak savings from head-only QT, with no reliable local speed gain | Profile the entire model, backward pass, optimizer, loading and transfers on the target. If convolution/activation traffic dominates, select and implement a supported deeper quantization path for that cost; if the head dominates, focus on its kernels/storage. | Report actual quantized/floating coverage and physical storage. Demonstrate end-to-end benefit at comparable quality; AMP or fake quantization alone does not complete this work. | +| Large normalized heads incur transient normalization/gradient storage and optimizer overhead | Preserve bounded preparation/backward fixes. Profile realistic head sizes, batching and fused update paths; consider optimizer-state reduction only if its footprint is limiting. | Fresh-process setup/steady-state measurements; allocated and reserved peaks; numerical, accumulation, optimizer-step and checkpoint regressions. Do not optimize only an isolated GEMM. | +| Compilation/CUDA graphs can improve steady state while increasing setup or reserved memory | Measure eager, compiled and graph modes separately. Qualify first-use tuning and cold setup on the target; reduce capture/tuning overhead only where it matters. | Include initialization and first step, replay correctness, total useful-work time and reserved memory. A warmed kernel speedup is insufficient. | +| Compiled floating-point DDP qualification emitted gradient/bucket stride mismatch, stochastic-depth recompilation-limit and hierarchical scalar-extraction graph-break warnings; throughput impact is unmeasured | Identify the affected parameter and layout before/after wrapping and backward; isolate compiler specialization and lazy hierarchy initialization with a small reproducer. Keep active qualification unchanged until measurements justify a fix. | Preserve gradient/update, stochastic-depth, checkpoint and DDP semantics; demonstrate reduced copies or graph fragmentation and repeatable warm end-to-end benefit. Warning disappearance alone is not acceptance. | +| Short hierarchical QT runs lose parent metrics; early BatchNorm sensitivity is substantial | Run longer paired budgets/seeds with working epoch statistics. Investigate precision/optimization sensitivity at fixed data and model state. Qualify any state-refresh or recipe change with its extra cost. | Time to comparable leaf/parent quality across seeds. Do not adopt automatic BatchNorm refresh from the mixed existing results. | +| Native dynamic INT8 ONNX uses MatMulInteger CPU fallback on CUDA; TensorRT rejects that representation | Qualify the explicit materialize-then-calibrate route first. If exact native quantizer semantics are required, implement a supported integer GPU lowering/kernel as a separate feature. | Actual provider/operator placement and end-to-end quality against the correct reference. Never silently label floating/fallback execution as integer GPU inference. | +| Large-head float export needs a profile-specific tolerance override | Continue numerical attribution with higher-precision references on the failing target/profile. Distinguish accumulated rounding from an export/operator defect. | Explicit score/parity and decision/metric evidence; retain the public default gate until a justified general change exists. | +| INT8 TensorRT engines are smaller but can be slower; local batch-64 results are near tied | Profile target batch sizes, transfers, tactics and head/backbone balance against FP16. Measure actual runtime memory and sustained throughput, not just serialized size. | Significant useful speed/cost or memory benefit with acceptable paired quality on the intended workload. | +| ARM kernels/runtime, thermal behavior and packaging are unverified | Prepare the actual ARM environment and inspect unsigned-activation CPU recipe placement; benchmark sustained batch-one inference including preprocessing. Adjust calibration/operator choices only from measured failures. | Repeated ARM measurements, all five metrics, memory within device limits, and an installed deployment example. x86 savings do not qualify ARM. | +| Shared storage/loading or CPU oversubscription may dominate training | Measure loader wait and host/device transfer on the real allocation. Tune explicit threads/workers, caching and prefetch/transfer options within resource limits. | Identical sample/order semantics, bounded host/shared memory, and improved end-to-end training rather than loader-only throughput. | + +If production size requires multiple GPUs, native QT DDP/FSDP is a separate +unsupported boundary to implement and qualify. Check parameter representation, +communication/reduction, sharding, optimizer state and distributed resume. Existing +floating DDP tests do not establish distributed QT support. + +## 4. Close development and integration bottlenecks + +| Dependency or gap | Required action | Done when | +| --- | --- | --- | +| Target-only optional runtimes and backend versions | Record supported combinations; preserve lazy imports and actionable errors; run installed-wheel validation after dependency/packaging changes. | Clean installation and intended kernels work on each claimed target without changing floating defaults. | +| Target inputs/models currently require explicit staging | Prepare a versioned deployment bundle with preprocessing, mappings, score semantics, calibration provenance, model hashes and a minimal inference example. Reuse current export manifests. | Another operator can reproduce the target run without private notebook state or this laptop's paths. | +| Continuous CPU/training handoff is incomplete | Connect the existing CPU command and representative training reports to configured runners and compact history. Add only the adapters needed for real collected evidence. | Failed and successful target runs retain attributable reports; history preserves CPU/GPU and training/inference measurement scopes. | +| Release storage/Pages has only local and simulated validation | After branch review/merge, configure Pages and target runner variables, then enable `ENABLE_BENCHMARK_HISTORY`. Run one live success, failure and publishing retry. | Stored asset readback, immutable run identity, correct public page and recoverable failure verified remotely. | +| Quality/resource acceptance is not automated | Define per-profile gates from practical paired evidence and the agreed few-points trade-off. Preserve all five metrics and uncertainty/repetition context. | A regression is visible and fails the appropriate check; unsupported/untested profiles cannot look production-qualified. | + +## 5. Completion and deferred work + +Finish with supported opt-in recipes for the three target regimes, reproducible +quality/efficiency evidence, functioning checkpoint/export/deployment contracts, +visible continuous results, and a final compatibility/packaging review. A negative +result should constrain the supported recipe or motivate a measured implementation +change; it must not be reported as a general speedup. + +EMA repair, unrelated optimizer/loss/augmentation comparisons, new dataset formats, +Birds/iNaturalist expansion and cosmetic dashboard work remain deferred. Broader +operator or distributed support becomes necessary when the target workload needs +it to achieve the stated goal, not as an unconditional expansion of this roadmap. diff --git a/docs/quantization-status.md b/docs/quantization-status.md new file mode 100644 index 00000000..c88ce11f --- /dev/null +++ b/docs/quantization-status.md @@ -0,0 +1,26 @@ +# Quantization branch status + +The goal is **not complete**: target-machine benefits have not been verified. +The maintained sources of truth are now: + +- [Branch roadmap](quantization-roadmap.md): ordered work, bottlenecks, dependencies + and completion gates. +- [Current benchmark findings](benchmarks.md): measured benefits, regressions and + interpretation limits. +- [Command index](../dev/benchmarks/README.md): training, inference and reporting. +- [Historical evidence](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md): detailed experiments, + unsuccessful approaches and numerical results retained for traceability. +- [Artifact retention](quantization-artifacts.md): what survived temporary cleanup + and how to restore evidence. + +Implemented locally: native INT8 Linear training with normalized symmetric heads; +optimizer/checkpoint safeguards; shared loading improvements; generic ONNX export; +explicit native-checkpoint materialization and calibration; CPU/TensorRT quality, +placement and resource commands; compact CPU/GPU records; opt-in release/Pages +publishing. The reporting service has not been qualified remotely. + +Not established: worthwhile end-to-end training gains on HPC/desktop targets, +integer convolution training, distributed native QT, exact native-quantizer GPU +ONNX execution, or ARM production performance. EMA repair remains deferred. +Further notebook experiments, dashboard expansion, unrelated feature comparisons, +Birds/iNaturalist and dataset-format work are not prerequisites for this branch. diff --git a/docs/quantization.md b/docs/quantization.md new file mode 100644 index 00000000..67b744da --- /dev/null +++ b/docs/quantization.md @@ -0,0 +1,139 @@ +# INT8 quantization: initial x86 backend + +This is an opt-in Python API for **static INT8 weights and UINT8 activation storage**, +using TorchAO PT2E. Actual quantized training with reduced memory and training +time now has an initial [CUDA model integration](quantized-training.md); see the [QT/loader probes](../dev/benchmarks/training.md#capacity-and-bottleneck-probes). It supports post-training calibration (PTQ) and +quantization-aware training (QAT). QAT uses fake quantization with float32 master +parameters/gradients; it does not promise integer backward computation or reduced +training memory. Converted inference executes native oneDNN integer Conv/Linear +kernels. This is separate from float16/bfloat16 AMP. + +Install the optional dependency in an explicitly selected backend environment: + +```bash +uv sync --extra cpu --extra quantization +# Existing CUDA environments: do not run a CPU sync; select their CUDA extra. +``` + +The first verified backend is x86 CPU with PyTorch 2.12 and TorchAO 0.17. +TorchAO is imported lazily. Ordinary training, prediction, checkpoint formats and +ONNX export are unchanged. The new API is in `mini_trainer.modeling.quantization`. + +## Calibration and inference + +```python +import torch +from mini_trainer.modeling.quantization import prepare_int8, load_int8 + +# model is a loaded floating-point mini_trainer model. All inputs below are +# float32 CPU batches AFTER the same preprocessing used for ordinary inference. +prepared = prepare_int8(model, example_batch) +with torch.no_grad(): + for images in training_calibration_batches: + prepared(images) +converted = prepared.convert() +converted.save( + "int8-model", + example_batch, + preprocessing={"recipe": "record the actual resize, scale and normalization"}, + calibration={"split": "train", "manifest_sha256": "record the actual manifest hash"}, +) +inference, coverage = load_int8("int8-model").lower(example_batch) +with torch.no_grad(): + scores = inference(example_batch) +``` + +Calibration must use training data, never held-out validation/test examples. +The caller supplies provenance; the API cannot infer the provenance of tensors. +The prepared PTQ graph is a calibration object, including when its mode is eval. +Convert it before evaluating held-out data. Conversion refuses unobserved or +nonfinite ranges, and does not modify the prepared model. + +Weights use symmetric per-channel int8; activations use affine per-tensor uint8. +The default `reduce_range=True` uses activation values 0..127 (seven effective +bits in eight-bit storage). This avoids intermediate saturation in oneDNN's +AVX2/non-VNNI kernels. Full range 0..255 is available with `reduce_range=False` +for a validated VNNI deployment target. The strict reference-versus-native +parity check remains enabled in both modes; tolerances are unchanged. + +This portability fix changes the default calibration/QAT range. Recreate older +full-range QAT checkpoints with `reduce_range=False` to preserve their recipe. +Existing exported full-range graphs are not silently recalibrated on load; +they still require compatible hardware and successful lowering parity. +New reduced-range recipes record the activation bounds explicitly. +Bias, normalization, score transforms and other non-linear operations may remain +floating point. The report includes the actual remaining operator inventory. +All captured Conv1d/Conv2d/Linear operations must receive weight and activation +annotations. Lowering fails if it cannot produce integer kernels or leaves +floating Conv/Linear kernels. It never labels a plain Q/DQ reference execution +as native integer inference. + +The bundle contains a reference `model.pt2` graph, checksum, input shape, class +metadata, structured output mapping, bit widths, dependency versions, +preprocessing/calibration provenance, and verified lowering coverage. Real calibration tensors are excluded from the saved program. Packing +is performed again on the deployment CPU. A reference graph alone is not an +accelerated runtime. Existing output directories are never overwritten. + +## Quantization-aware training + +```python +prepared = prepare_int8(model, example_batch, qat=True) +optimizer = torch.optim.AdamW(prepared.parameters(), lr=1e-4) +prepared.train() +for images, targets in training_batches: + optimizer.zero_grad() + loss = criterion(prepared(images), targets) + loss.backward() + optimizer.step() + +prepared.freeze_observers() # Optional: hold learned ranges fixed for later steps. +prepared.eval() # Evaluation does not update QAT ranges or BatchNorm. +with torch.no_grad(): + scores = prepared(validation_batch) +converted = prepared.convert() +``` + +Construct the optimizer **after** preparation. Save `prepared.state_dict()` and +the optimizer/scheduler/scaler states. Restore into an identically prepared model, +then restore optimizer state. Observer ranges, fake-quant flags and the explicit +freeze flag are part of the state. The recipe is checked on restoration. This is +not an ordinary `Classifier.build(weights=...)` checkpoint: automatic reconstruction +through `mt_train`/`mt_predict` remains a subsequent integration step. + +QAT runs can use `train_one_epoch` with an appropriate criterion, disabled EMA, +and no embedding-dependent regularizer. Captured graphs neither consume ambient +supervision nor populate `EmbeddingContext`. Train/eval switching covers dropout +and BatchNorm; arbitrary Python training branches are specialized by capture. +Autoregressive teacher-forcing/sampling requires a separate training integration +and is not currently a supported QAT claim. Capture failures propagate explicitly. + +## Coverage and next increments + +Inputs currently have a fixed captured batch and image shape. All batches must +match it; pad and slice final inference batches, or prepare a separate shape. +The original model's parameters, modes and caches are preserved by preparation. +Functional linears, weight parametrization, hierarchical aggregation and class +masks are included in capture; this does not rely on a backbone allowlist. + +Focused tests exercise flat, hierarchical, conditional and independent heads, +real gradients, exact controlled QAT/AdamW continuation, held-out-data isolation, +integer operator execution, checksums and artifact reload. A synthetic oracle +exercises QAT through the actual training loop and checks integer predictions. + +Still required: user-facing checkpoint/CLI integration, dynamic batch support, +MNIST/Blair reports, broader backbone/operator coverage, GPU quantization, +ONNX/runtime conversion, and lower-bit profiles. Model quality, artifact size, +memory and latency need measured comparisons; no speedup or quality benefit is +claimed by passing compatibility tests. EMA remains unsupported. + +Run focused checks without changing the installed environment: + +```bash +OMP_NUM_THREADS=1 bash dev/check.sh test tests/quantization/test_quantization.py +``` + +Backend references: [PT2E x86 quantization](https://docs.pytorch.org/ao/stable/pt2e_quantization/pt2e_quant_x86_inductor.html), +[QAT workflow](https://docs.pytorch.org/ao/stable/pt2e_quantization/pt2e_quant_qat.html). +Strict graph capture is intentional: the installed backend otherwise misses +functional-linears' source metadata. Explicit `lower_pt2e_quantized_to_x86` +provides native kernels without relying on a compiler silently optimizing Q/DQ. diff --git a/docs/quantized-training-validation.md b/docs/quantized-training-validation.md new file mode 100644 index 00000000..fc838556 --- /dev/null +++ b/docs/quantized-training-validation.md @@ -0,0 +1,58 @@ +# Quantized training validation contract + +This document describes the checks that must remain valid while quantization is +extended. Current results live in [benchmark findings](benchmarks.md); planned +work and target acceptance live in the [branch roadmap](quantization-roadmap.md). +Detailed historical runs are retained in the [experiment archive](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md). + +## Model and dataset coverage + +The primary model is EfficientNetV2-S with a symmetric hidden layer and normalized +`Classifier` or `HierarchicalClassifier`. Compare both heads on the same reviewed +Blair splits, with full and parameter-frozen training reported separately. +Synthetic oracle and MNIST cases verify fast integration. Synthetic 10k/100k-class +heads verify scaling; they do not replace real-data quality measurements. +Keep full million-class experiments off the laptop. + +## Required invariants + +- Quantization reports actual selected operators, physical weight/activation + storage and floating operations. Current native training covers Linear weights + and saved inputs; convolutions, gradients and optimizer state remain floating. +- Ordinary floating defaults, shape/dtype/device contracts, class ordering, + normalized and hierarchical score semantics remain stable. +- SGD, AdamW and MuonAuxAdamW retain optimizer-step, scheduler, AMP-overflow, + accumulation, compilation and checkpoint behavior. `_step_count` is part of + this coordination; do not replace it without validating each optimizer's semantics. +- Controlled checkpoint continuation is tested with fixed ordering and disabled + stochastic transforms. Arbitrary resume does not promise identical trajectories: + general RNG/sampler state is not fully checkpointed. +- EMA remains disabled and unsupported; retain its explicit warning and strict + expected-failure regression. DDP/FSDP support for native QT is not implied by + floating CPU distributed tests. +- Native integer export and materialize-then-calibrate export are different + numerical contracts. Verify output semantics, preprocessing, input identities, + class mappings, operator placement and paired quality for the selected route. +- Worker selection respects available affinity/quota/allocation information; + unrestricted shared-node contention still requires explicit conservative counts. + +## Checks and interpretation + +Use the [development checks](../dev/README.md) and grouped +[test suite](../tests/README.md). CPU checks cannot establish GPU correctness. +Intentional CUDA checks require explicit visibility and the documented flags; +optional runtimes/backbones may require a separately prepared environment. +Do not implicitly synchronize the working CUDA installation. + +Benchmark quality uses `mini_metrics` Macro-F1, Macro-Recall, Macro-Precision, +Coverage and Theil's U at every reported hierarchy level. Keep paired seeds, +negative results, confidence/score semantics and undefined metrics visible. +Only compare resource measurements collected under suitable isolated conditions. +A few points of degradation may be acceptable when justified by a material +speed/cost or memory benefit. Record the per-profile acceptance limits before +qualification; integer execution alone is not acceptance. + +The intended execution targets remain HPC GPU training on A40/A100/B300-class +systems with EPYC hosts, local training/fine-tuning and ONNX GPU inference on Spark +or the intended RTX desktop, and ONNX CPU inference on Raspberry Pi/ARM. +Support claims require evidence from the corresponding hardware and workload. diff --git a/docs/quantized-training.md b/docs/quantized-training.md new file mode 100644 index 00000000..9dd7f51c --- /dev/null +++ b/docs/quantized-training.md @@ -0,0 +1,182 @@ +# CUDA INT8 training integration + +The opt-in training path stores eligible Linear weights and saved linear inputs +in INT8 and uses integer matrix products for forward, input gradients and weight +gradients. It retains no floating-point master copy of those weights. Gradients, +optimizer state, biases, activation normalization, convolutions and auxiliary regularization +remain floating point. This is separate from fake-quantized QAT and x86 PTQ. + +Install the optional `quantization` extra while explicitly retaining the intended +PyTorch CUDA backend, as described in the README. The current implementation uses +TorchAO's experimental Triton kernels. It has been exercised on an RTX 3080 Ti; +CPU preparation and checkpoint inspection do not establish CPU execution support. +See the [validation audit](quantized-training-validation.md) for current tests, +measured training/loading benefits and the limits of those results. + +```bash +mt_train -i /path/to/data --device cuda --quantized-training --dtype float16 --cache cpu --cache-workers 0 +``` + +`--quantized-training` prepares weights before building the optimizer and logs +coverage. `--cache-workers 0` makes cache construction synchronous; its selection +is separate from DataLoader workers. Ordinary training defaults are unchanged. + +The Python API also supports explicit module selection: + +```python +import torch +from mini_trainer.modeling.quantized_training import prepare_quantized_training + +# Load floating weights and move the model to its intended device first. +coverage = prepare_quantized_training(model) # in place, before optimizer creation +# Or: prepare_quantized_training(model, module_names=["encoder.projection"]) +optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3) +``` + +The returned recipe lists quantized modules, skipped operations, remaining +floating-point parameters and physical versus reference weight storage. Automatic +selection covers ordinary `nn.Linear` modules, including their functional use by +Classifier heads. It preserves shared weights when every owner is selected. +Initial conversion of large weights processes row chunks of at most 4,194,304 +elements (or one row when wider), limiting the quantizer's floating temporaries. +This preserves deterministic INT8 codes and row scales; it does not reduce the +storage needed for source weights, validation, gradients or optimizer states. +Row-wise PyTorch weight normalization (`dim=0`) quantizes the direction parameter +while retaining its scalar magnitude per output row in floating point. Effective +normalized weights reuse the integer codes with new row scales; normalization +backward applies its Jacobian to the approximate Linear gradient. No floating +weight matrix is retained for this operation. Other parametrizations, normalization +dimensions and weights shared with an unselected operation stay floating point +and are reported. Explicit unsupported +selections and models with no eligible weights fail before changing weights. +Quantizing the hidden linear layer of a convolutional classifier does not make +its convolutions integer operations. + +CUDA execution supports batched inputs, bias, masked classifier rows, +single-sample inference and float16/bfloat16 autocast with float32 parameters. +Float32 optimizer parameters avoid the FP16 AdamW epsilon underflow discussed in +the developer probe. Eager SGD, AdamW and the repository's MuonAuxAdamW update +paths are covered; fused optimizer variants are not established. Quantized +regularization uses a differentiable floating view of the represented weights. + +Eager CUDA SGD/AdamW-style `add_` and `addcdiv_` updates fuse dequantization, +the weight update and stochastic requantization over the underlying storage +tensors. This kernel compiles on first use even when the outer optimizer is +eager. It retains INT8 codes and row scales, advances tensor version counters, +and preserves explicit intermediate precision casts. Scalar tensor weight decay +rescales rows without another stochastic rounding pass. CPU inspection/update +tests use the ordinary floating calculation and copy path. The fused row kernel +covers matching FP32/FP16/BF16 update tensors and rows up to 16,384 elements; +broadcasting, mixed dtypes and wider rows retain the ordinary update path. +No floating master weight is retained by either path. The new kernel uses CUDA +RNG seeds with Triton stochastic rounding, so exact trajectories differ from the +earlier floating update even when starting from the same seed. + +Matrix products reuse TorchAO's INT8 kernel with a separate local tuner. CUDA +graph timing avoids the default tuner's 256 MiB cache-flushing allocation, and +selected configurations are cached on disk. TorchAO's global operators and tuner +are unchanged. The explicit update and matrix operators provide fake execution +implementations for model compilation. + +`--compile-optimizer` supports tensor-learning-rate FMA updates and functional +stochastic requantization. Floating-to-INT8 copies use a fused row kernel for +matching CUDA FP32/FP16/BF16 tensors with at most 16,384 columns, returning fresh +codes/scales before the final storage mutation. Eager and compiled rounding can +follow different random trajectories even with a matching seed. Transient floating +updates remain; compilation compatibility does not imply a speedup. + +`--compile-optimizer --optimizer-cudagraphs` opts into optimizer graph replay. +During AOT fake-tensor tracing, updates expose floating arithmetic followed by +functional requantization and storage copies. Eager native updates are retained. +Expected failed kernel-tuning candidates release their exception tracebacks +promptly so temporary tensors do not outlive graph pool tracking during first +use; unexpected kernel errors still propagate. +Learning rates stay on CUDA during replay; checkpoints retain numeric values and +explicit non-default rate precision. This leaves the AMP gate, scheduler and +MuonAuxAdamW outer counter in their existing roles. The training loop explicitly +marks each graph iteration before the model runs, keeping backward gradient +buffers alive until the optimizer consumes them. Custom training loops must call +[`torch.compiler.cudagraph_mark_step_begin()`](https://docs.pytorch.org/docs/2.12/generated/torch.compiler.cudagraph_mark_step_begin.html) +before each training iteration when +combining compiled models with optimizer graph replay. See the +[optimizer graph requirements](../dev/README.md#optimizer-cuda-graphs), including +native fused float32-rate restrictions. The native fused optimizer tests use +floating parameters; they do not establish native fused updates of INT8 weights. + +## Checkpoints and inference + +Prepared models include their recipe in `state_dict`. Ordinary `mt_train` +checkpoints retain INT8 parameter storage, and `Classifier.build(weights=...)` +restores the parameter types before loading weights. Known tensor classes are +allowed only within a scoped `weights_only=True` load. To resume through +`mt_train`, enable `--quantized-training` again so optimizer construction sees +the correct parameters. Existing stochastic-resume limitations still apply: +the trainer does not generally persist sampler or RNG state. + +For a custom architecture outside `Classifier.build`: + +```python +from mini_trainer.modeling.quantized_training import load_training_weights, restore_quantized_training + +state = load_training_weights("weights.pt", map_location="cpu") +restore_quantized_training(model, state) +model.load_state_dict(state) +``` + +Use the same architecture and intended dtype. This restores model state; create +and restore optimizer/scheduler/scaler state in their normal order separately. +The same model supports CUDA inference with `eval()` and `inference_mode()`. +An opt-in [ONNX export path](onnx.md#native-int8-training-checkpoints) captures +the integer forward using a full-FP32 CUDA reference and verifies ONNX Runtime CPU +parity. Target-provider performance remains unverified. Checkpoint averaging, +DDP/FSDP, quantized activation normalization and integer convolution training +are not established for this path. Distributed training and +EMA are rejected by the training entry point. + +## Model compilation modes + +Model compilation accepts `--compile --compile-mode reduce-overhead` (or +`compile=True, compile_mode="reduce-overhead"` in Python). This is opt-in and +independent of optimizer compilation. The benchmark runner records the selected +mode. See [compilation guidance](../dev/README.md#model-compilation) for the other +modes and measurement requirements; selecting a mode does not establish a speedup. + +Embedding publication preserves context side effects across compilation, with +normalized weights beside their Linear consumer. Context remains shared, not +thread/task-local. AMP fusion can change rounding and INT8 activation bins; +eager and compiled trajectories are not promised to be bitwise identical. +Compiler cache keys include backend source and tensor metadata so hidden backward +changes invalidate cached graphs. + +## Evidence and remaining work + +See [current findings](benchmarks.md), [validation contracts](quantized-training-validation.md) +and [the roadmap](quantization-roadmap.md). These own measured benefits, negative +results and remaining hardware/quality work. Use [the training guide](../dev/benchmarks/training.md) +for reproducible commands. Kernel speedups do not establish real-model convergence +or end-to-end efficiency. + +Row-wise normalization tests cover represented-value forward/backward, signed +scales, zero magnitudes, saved storage, masked inference and checkpoint restoration. +Initial zero direction rows are rejected before mutation. Whole-run memory includes +initialization, gradients and optimizer storage as well as compressed parameters. + +## Large-class accumulator bounds + +Input-gradient products contract over the output class count. For contractions +above 131,071, the backend now combines bounded INT32 dot products in INT64 before +converting and scaling the result. This prevents finite but saturated gradients +for large vocabularies. Shorter contractions keep the existing tuned kernel. +ONNX export also bounds integer partial products and combines them in INT64. +See the [arithmetic regression and limits](https://github.com/asgersvenning/mini_trainer/blob/f5c69e7cab2bfde8a5467026b293858b93e628f9/docs/archive/benchmark-history.md#long-contraction-int8-accumulator-correctness). +A million-output Linear gradient test does not establish that a full million-class +EfficientNetV2 training configuration fits the available GPU; parameter, +initialization, gradient and optimizer storage still require separate measurement. + +## Implementation layout + +Public APIs remain in `modeling.quantized_training` and `modeling.quantization` +(the separate PTQ/QAT backend). Private native INT8 code lives in +`modeling/_quantized_training/`: tensor dispatch in `__init__.py`, plus `matmul.py`, +`normalization.py`, `update.py` and ONNX translation in `onnx.py`. Keeping the backend +package at its original module path preserves serialized `TrainingWeight` identities. diff --git a/docs/roadmap.md b/docs/roadmap.md index 3400c9b5..6ca554bb 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,9 @@ # Repository strengthening roadmap +The active quantization feature branch has a separate +[execution roadmap](quantization-roadmap.md) covering measured bottlenecks, +target-machine dependencies, integration gates and deferred work. + This is the implementation order. Each increment should leave the existing default training and prediction interfaces working and include its own validation evidence. Items below are planned unless explicitly marked delivered. @@ -88,49 +92,97 @@ explicit upload commands or a serving deployment. ## 4. Training efficiency and augmentation -The primary implementation target is **deeper quantization in training and inference**. -Loader hardening, float16/bfloat16 AMP and benchmark infrastructure do not complete -that target. The implementation and comparison plan is in -[training feature validation](training-feature-validation.md). - -Deliver quantization-aware training and post-training inference quantization as -separate opt-in capabilities, recording actual weight/activation bit widths, -calibration data, backend kernels, checkpoint/resume and export/runtime support. -FP8 or other reduced-precision compute is a separate hardware-dependent profile. -Keep unsupported model/backend combinations explicit; do not silently run an -unquantized model while reporting a quantized result. EMA support is not a gate -for this work while the feature is declared defunct. - -Improve the default augmentation pipeline after paired task-aware experiments, -retaining a reproducible legacy recipe. Respect label semantics: digit tasks, -color-based synthetic tasks and biological imagery need different invariances. -Document uint8 input and augmentation-before-preprocessing behavior accurately. - -Benchmark foundation delivered: the [continuous dataset benchmark suite](benchmarks.md) -uses a synthetic oracle, MNIST and hierarchical Blair, with CPU and GPU/AMP/cache -profiles, explicit split/provenance records, Actions summaries and retained artifacts. -This is a verifiable milestone, not a CPU-only scope boundary. Add durable hosted -history, repeated comparisons and coverage of the remaining training features next. - -Next establish repeatable measurements for loader wait time, images/second, host/GPU -memory, and validation quality on fixed configurations. Existing code already uses -autocast, optional compilation, persistent workers, cache modes, and DDP spawn handling. - -Add opt-in loader controls only where measurements justify them (for example, -prefetching and transfer overlap), with coverage for zero workers, cache modes, -deterministic seeding, and distributed sampling. Extend augmentation through the -existing builder interface, documenting label and dtype requirements. - -Evaluate lower-precision training, quantization-aware training, and inference -quantization as distinct paths. Keep current defaults and checkpoint compatibility; -validate optimizer/resume behavior (EMA is deferred), numerical stability, hardware support, export -compatibility, memory, throughput, and quality before recommending a configuration. - -Acceptance: reproducible baseline and comparison results, explicit supported hardware -and backends, opt-in configuration, and no regression in default training behavior. +Delivered: opt-in native CUDA INT8 Linear training, x86 PTQ/QAT inference, +checkpoint/export integration, bounded preparation/normalization storage, and +allocation-aware loader hardening. Defaults remain floating point. EMA is deferred; +native QT DDP/FSDP is unsupported. + +The [quantization roadmap](quantization-roadmap.md) owns remaining performance, +quality and deployment work. [Measured findings](benchmarks.md) distinguish local +memory savings from mixed speed/convergence results; HPC, desktop/Spark and ARM +qualification remain open. AMP, fake quantization and CPU smoke tests do not +complete the deeper quantization objective. + +After quantization, evaluate a task-aware augmentation recipe against the legacy +pipeline. Preserve label semantics, input dtype, optimizer/accumulation/resume +behavior and existing builder extension points. Add loading controls only when +whole-training measurements demonstrate a benefit. + +Acceptance: opt-in supported recipes, reproducible paired quality/resource evidence, +and unchanged default behavior. Use [training feature validation](training-feature-validation.md) +for the deferred optimizer, loss and augmentation comparisons. + +### Optional dataset preparation for scalable loading + +Target: remove repeated small-file access bottlenecks on shared storage through an +optional preprocessing command that prepares an indexed, sharded dataset for the +existing training and prediction loaders. This is an efficiency representation of +already-supported inputs, not a requirement to migrate source datasets or adopt a +new training API. Keep the public design independent of a particular provider. + +- Reuse the current source/metadata adapters. Cover every currently supported + input, flat and hierarchical labels, multilabel targets, supplied splits, + class ordering and sample identity. Retain lazy loading, inference and supported + single-process/DDP behavior. Define a compatibility matrix before implementation; + do not silently drop cases that are inconvenient for a shard backend. +- Preserve original encoded image bytes by default and apply the existing decoder, + resize, transforms and hooks at loading time. Do not bake stochastic augmentation + into prepared data. Any later materialized-preprocessing option must be explicit, + versioned and checked against the requested runtime preprocessing. +- Store a versioned manifest with source provenance, sample-to-shard index, labels, + splits, class mappings and integrity information. Support bounded, resumable + preparation with atomic publication of completed artifacts and explicit errors + for missing, changed or corrupt samples; never silently skip them. +- Evaluate indexed uncompressed TAR shards before inventing a container format. + Preserve current sampler order and epoch coverage in the first implementation. + Treat locality-aware or streaming shuffles as separate opt-in behavior changes, + with explicit DDP partitioning, equal-step and checkpoint/resume contracts. +- Allow direct reads of prepared shards and optional staging to verified node-local + storage. Share a byte-bounded cache across ranks/workers on each node, coordinate + downloads, protect in-use shards and publish verified cache entries atomically. + Measure cache churn under the actual sampler; packing files alone does not + guarantee efficient random access or eliminate cold-read latency. +- Keep implementation within the existing metadata/reader boundaries, using a + focused preparation/storage module where needed. Expose it through the normal + CLI and Python configuration paths, keep new dependencies optional, and preserve + the original uncached path and defaults. + +Deliver in bounded increments: compatibility fixtures and a preparation/loader +round trip; indexed shards; optional shared local staging/cache. A separate small +loader improvement may add bounded concurrent encoded-byte reads within each +batch, preserving sample order and hook execution without multiplying process-local +metadata. Benchmark that independently of changing storage representation. + +Acceptance: byte-identical decoded inputs for the default representation, identical +labels/splits/class ordering and sampler coverage, supported training/prediction and +DDP restoration checks, corruption/interrupted-preparation recovery, and bounded +memory/disk use. Compare first-pass and repeated-pass end-to-end throughput on a +representative working set, including conversion/staging time, validation and +figures. Report the number of epochs needed to amortize preparation; a warmed tiny +subset is insufficient evidence. Implementation and GPU qualification remain open. ## 5. mini_metrics and continuous model evaluation +Planned target: economical continuous benchmarks with a GitHub audit dashboard. +Connect hosted CPU checks and small, on-demand GPU runs to durable report history +through an independent publisher. Use fixed representative subsets of existing +datasets, preserve supplied splits, and compare baseline/candidate under matching +hardware and workload conditions. Large distributed qualification and production +training remain separately triggered activities, outside the continuous schedule. + +Keep allocation credentials in a trusted controller isolated from candidate code, +pull requests and report publishing. Restrict profiles, revisions, dataset access, +concurrency and allocation duration; enforce a persistent spending budget, reconcile +ambiguous submissions before retrying, and disable automatic time extension. +Provider/account configuration and infrastructure review details remain local. + +Acceptance: dry-run and mocked lifecycle tests cover interruption, duplicate +submission, budget exhaustion and credential isolation before a capped live trial. +Then demonstrate cancellation/cleanup and auditable success, failure and publishing +retry records before enabling a schedule. Missing runs must remain visible; partial +GPU allocations must not imply full-device or distributed performance qualification. +Build on the existing [reporting integration](../dev/benchmarks/reporting.md). + Document measured effects of MuonAuxAdamW versus AdamW/SGD, `normalized`, EMLACrossEntropy, class-weight distribution regularization, automatic label smoothing, and hierarchical versus flat classifiers. Use the same datasets/splits and paired @@ -152,6 +204,17 @@ class order, score meaning, confidence thresholds, missing classes, and hierarch labels. Verify installed mini_metrics APIs before writing an adapter. Keep evaluation optional and preserve existing prediction formats and research scripts. +Deferred: consolidate flat and hierarchical inference behind one public CLI. +`mt_hpredict` already shares the generic prediction CLI and has an explicit flat +head route; use that existing functionality while training qualification proceeds. +Before refactoring, assess whether stored head/taxonomy metadata can reliably +select the builder and result collector, or whether weights need additional +versioned configuration. Preserve explicit CLI overrides, legacy checkpoints, +existing entry points, class ordering and prediction/mini_metrics output contracts. +Acceptance: installed CLI tests cover flat and hierarchical weights, including +older artifacts with missing metadata and actionable handling of ambiguous cases. +This is not a prerequisite for DDP qualification or the production training run. + Build the model zoo around versioned manifests linking immutable weights, configuration, preprocessing, dataset/split identity, code/dependency versions, and evaluation results. Use fixed held-out data and seeds; distinguish threshold selection from test evaluation. @@ -163,9 +226,41 @@ results trace back to exact inputs, and metric changes cannot silently rewrite b ## 6. Additional dataset formats (low priority) +Future ingestion support must cover both training and inference through shared +source adapters. Keep discovery (sample identity, paths, supplied labels and +splits) separate from training partitioning and model-vocabulary indexing. Start +with a compatibility matrix of existing formats and flat/hierarchical/unlabelled +inputs; add formats incrementally without duplicating prediction-only readers. + +Follow-ups identified during the bounded `auto_find_images` review: + +- Clarify `create_taxonomy` / `select_levels`: an integer currently means an + inclusive deepest-rank index, whereas callers such as `create_metadata` pass + `len(cls2idx)` as a count. The inference caller now uses explicit rank indices. + Audit remaining callers before changing the shared signature; test actual rank + selection rather than mocking the whole taxonomy adapter. Separate API/cache + retrieval failures from local rank-selection and mapping errors in diagnostics. + +- Define an explicit input-layout/split policy for ambiguous folder layouts and + reconcile prediction's separate `data_index` and source-selection paths. +- Specify taxonomy provenance and missing-ancestor handling independently of + model vocabulary. Preserve valid species ground truth without treating unknown + ancestors as real unseen classes; cover the collector/mini_metrics contract + before changing the existing GBIF fallback. +- Audit image probing on read-only datasets: `is_image` currently opens files in + `r+b` mode and can silently exclude readable images without write permission. + +The focused discovery fix removes training metadata construction and vocabulary +filtering from folder enumeration; these broader policies remain deferred. + Add formats through existing metadata and reader boundaries after the higher-priority interfaces stabilize. Require format-independent class ordering, split handling, multilabel behavior, lazy loading, and useful errors. Keep format dependencies optional. Acceptance: tiny fixtures exercise each new format through training and prediction without changing existing format detection or defaults. + +Follow-up from candidate-filter inference validation: `classification_module` +caches an empty attribute name when passed a bare classifier head, causing a +subsequent lookup to fail. Normal built backbone models are unaffected. Cover +bare-head lookup separately rather than expanding the class-list CLI change. diff --git a/docs/training-feature-validation.md b/docs/training-feature-validation.md index 1d5314a0..762fa2e0 100644 --- a/docs/training-feature-validation.md +++ b/docs/training-feature-validation.md @@ -4,7 +4,18 @@ This is planned work. Existing CPU/GPU benchmark results establish a baseline; they do not measure the benefit of the features below. EMA is temporarily nonfunctional and excluded from these experiments; repair is deferred. -## Quantization is the primary implementation target +## Actual quantized training is the primary implementation target + +The priority is now QT that lowers training memory and increases speed, together +with faster data loading for floating-point and quantized workloads. PTQ/QAT do +not satisfy that objective. See the [QT and loader probes](../dev/benchmarks/training.md#capacity-and-bottleneck-probes); +an initial [model/trainer integration](quantized-training.md) is available, while +broader optimizer/resume coverage, convergence and end-to-end measurement remain +requirements, not optional follow-ups. + +The initial [INT8 PTQ/QAT Python backend](quantization.md) is implemented on the +`quant` branch. Its CPU tests establish a training-to-integer-inference path; +user-facing checkpoint integration, other backends and quality studies remain open. Deliver two distinct paths through the existing builders, checkpoint and export interfaces, with optional dependencies: diff --git a/mini_trainer/data/_prefetch.py b/mini_trainer/data/_prefetch.py new file mode 100644 index 00000000..9a1ab86c --- /dev/null +++ b/mini_trainer/data/_prefetch.py @@ -0,0 +1,78 @@ +"""One-batch CUDA transfer lookahead, independent of model precision.""" + +import torch +from torch.utils.data import DataLoader + + +def _map_tensors(batch, operation): + if isinstance(batch, torch.Tensor): + return operation(batch) + if isinstance(batch, dict): + return {key: _map_tensors(value, operation) for key, value in batch.items()} + if isinstance(batch, tuple) and hasattr(batch, "_fields"): + return type(batch)(*(_map_tensors(value, operation) for value in batch)) + if isinstance(batch, (tuple, list)): + return type(batch)(_map_tensors(value, operation) for value in batch) + return batch + + +class CUDAPrefetchLoader(DataLoader): + """DataLoader whose opt-in iterator yields tensors on the requested CUDA device. + + The sampler and worker lifecycle remain those of DataLoader. Only one extra + device batch is staged. Preprocessing and augmentation stay on the caller's + compute stream. CPU hooks may run one batch earlier than without lookahead. + """ + + def __init__(self, *args, device, **kwargs): + self.transfer_device = torch.device(device) + if self.transfer_device.type != "cuda": + raise ValueError("CUDA batch prefetch requires a CUDA target device.") + super().__init__(*args, **kwargs) + + def __iter__(self): + return self._prefetch(super().__iter__()) + + def _prefetch(self, source): + stream = torch.cuda.Stream(device=self.transfer_device) + stream.wait_stream(torch.cuda.current_stream(self.transfer_device)) + sentinel = object() + + def preload(): + batch = next(source, sentinel) + if batch is sentinel: + return sentinel + producer = torch.cuda.current_stream(self.transfer_device) + + def transfer(tensor): + if tensor.device.type == "cuda": + if tensor.device != stream.device: + raise ValueError("CUDA prefetch cannot stage tensors from another CUDA device.") + stream.wait_stream(producer) + tensor.record_stream(stream) + return tensor.to(self.transfer_device, non_blocking=True) + + with torch.cuda.stream(stream): + return _map_tensors(batch, transfer) + + pending = preload() + while pending is not sentinel: + current = torch.cuda.current_stream(self.transfer_device) + # Wait only for this batch, before queueing the next copy. + current.wait_stream(stream) + + def record(tensor): + tensor.record_stream(current) + return tensor + + batch = _map_tensors(pending, record) + error = None + try: + pending = preload() + except Exception as caught: + # Deliver the already loaded batch before surfacing a later + # reader failure, as ordinary sequential iteration would. + error = caught + yield batch + if error is not None: + raise error diff --git a/mini_trainer/data/_workers.py b/mini_trainer/data/_workers.py index e0e2bdc4..ae5cab91 100644 --- a/mini_trainer/data/_workers.py +++ b/mini_trainer/data/_workers.py @@ -1,4 +1,62 @@ import os +import re +from pathlib import Path + + +def _read_text(path): + try: + return path.read_text() + except (OSError, UnicodeError): + return "" + + +def _cgroup_cpu_count(): + """Smallest visible v1/v2 bandwidth quota, including ancestor limits.""" + groups = {} + for line in _read_text(Path("/proc/self/cgroup")).splitlines(): + fields = line.split(":", 2) + if len(fields) == 3: + for controller in fields[1].split(","): + groups[controller] = fields[2] + limits = [] + for line in _read_text(Path("/proc/self/mountinfo")).splitlines(): + left, separator, right = line.partition(" - ") + fields, filesystem = left.split(), right.split() + if not separator or len(fields) < 5 or len(filesystem) < 3: + continue + if filesystem[0] == "cgroup2": + group = groups.get("") + filenames = ("cpu.max",) + elif filesystem[0] == "cgroup" and "cpu" in filesystem[2].split(","): + group = groups.get("cpu") + filenames = ("cpu.cfs_quota_us", "cpu.cfs_period_us") + else: + continue + if group is None: + continue + root, mount = (Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) for value in fields[3:5]) + group = Path(group) + if not all(path.is_absolute() and ".." not in path.parts for path in (root, mount, group)): + continue + try: + relative = group.relative_to(root) + except ValueError: + # A cgroup namespace can expose paths relative to its own root. + relative = group.relative_to("/") + current = mount / relative + while True: + values = " ".join(_read_text(current / name) for name in filenames).split() + if len(values) == 2: + try: + quota, period = map(int, values) + if quota > 0 and period > 0: + limits.append(quota // period) + except ValueError: + pass # v2 "max", or malformed/unavailable quota data. + if current == mount: + break + current = current.parent + return min(limits) if limits else None def _available_cpu_count() -> int: @@ -14,7 +72,18 @@ def _available_cpu_count() -> int: counts.append(len(os.sched_getaffinity(0))) except (AttributeError, OSError, NotImplementedError): pass - return min(counts) if counts else (os.cpu_count() or 0) + if not counts: + counts.append(os.cpu_count() or 0) + quota = _cgroup_cpu_count() + if quota is not None: + counts.append(quota) + try: + allocated = int(os.environ.get("SLURM_CPUS_PER_TASK", "")) + if allocated > 0: + counts.append(allocated) + except ValueError: + pass + return min(counts) def _default_worker_count(cap: int, reserve: int = 4, minimum: int = 0) -> int: diff --git a/mini_trainer/data/io.py b/mini_trainer/data/io.py index 15178222..d4cad28b 100644 --- a/mini_trainer/data/io.py +++ b/mini_trainer/data/io.py @@ -3,11 +3,13 @@ import operator import os import warnings +from collections import deque from collections.abc import Callable, Iterable, Iterator, Sequence from concurrent.futures import ThreadPoolExecutor +from contextlib import closing from enum import Enum -from queue import Queue -from threading import Thread +from functools import lru_cache +from itertools import batched from typing import Any, TypeVar, cast import numpy as np @@ -110,6 +112,28 @@ def _pil_to_torch_interp(interp: int) -> InterpolationMode: return m.get(interp, InterpolationMode.BILINEAR) # type: ignore +@lru_cache(maxsize=32) +def _nearest_indices(source_size, target_size): + # Match ATen's legacy nearest mapping: float32 scale, floor, then clamp. + # Cache only one-dimensional coordinates, never full image-sized maps. + scale = np.float32(source_size) / np.float32(target_size) + indices = (np.arange(target_size, dtype=np.float32) * scale).astype(np.int64) + np.minimum(indices, source_size - 1, out=indices) + return torch.from_numpy(indices) + + +def _resize_nearest_uint8(image, height, width): + if image.shape[-2:] == (height, width): + return image + rows = _nearest_indices(image.shape[1], height) + columns = _nearest_indices(image.shape[2], width) + # Decoders return interleaved RGB storage. Gather whole rows/pixels before + # restoring the contiguous CHW layout produced by torchvision resizing. + return ( + image.permute(1, 2, 0).index_select(0, rows).index_select(1, columns).permute(2, 0, 1).clone(memory_format=torch.contiguous_format) + ) + + class ReadAndResize: """Callable class to read and resize images from paths.""" @@ -136,7 +160,17 @@ def __call__(self, path: str | tuple[str, ...]) -> torch.Tensor: except Exception as e: e.add_note(f"Image path: {path}") raise - img = TF.resize(img, size=[self.h, self.w], interpolation=self.interp, antialias=self.antialias) + if ( + self.interp == InterpolationMode.NEAREST + and img.device.type == "cpu" + and img.dtype == torch.uint8 + and img.ndim == 3 + and 0 < min(self.h, self.w) + and max(self.h, self.w) <= 4096 + ): + img = _resize_nearest_uint8(img, self.h, self.w) + else: + img = TF.resize(img, size=[self.h, self.w], interpolation=self.interp, antialias=self.antialias) if img.dtype != self.dtype: img = self.converter(img) return img.to(self.device) @@ -339,6 +373,49 @@ def _infer_numeric_dtype(seq) -> Any: return object +class _DirectBatchIndices(list): + """Index list for the repository collator, which accepts stacked tensors.""" + + +def _shared_batch_buffer(template, shape): + # Match PyTorch's worker collator: allocate the final IPC storage directly, + # rather than stack/gather locally and copy it when the queue shares it. + storage = template._typed_storage()._new_shared(math.prod(shape), device=template.device) + return template.new(storage).resize_(shape) + + +class _FetchedBatch(list): + """Sample views for standard collators, with the already-stacked batch attached.""" + + def __init__(self, data, *, sample_views=True): + self.data = data + self._sample_views = False + super().__init__() + if sample_views: + self._materialize() + + def _materialize(self): + if not self._sample_views: + data = self.data + super().extend(data.unbind(0) if isinstance(data, torch.Tensor) else zip(*(value.unbind(0) for value in data))) + self._sample_views = True + + def __getitem__(self, index): + # An external default collator starts with batch[0]. Materialize here + # if it reuses our tagged batch sampler with a different collator. + self._materialize() + return super().__getitem__(index) + + def __iter__(self): + self._materialize() + return super().__iter__() + + def __len__(self): + if self._sample_views: + return super().__len__() + return len(self.data) if isinstance(self.data, torch.Tensor) else len(self.data[0]) + + class LazyDataset(torch.utils.data.Dataset): """A general lazy dataset which calls func on items to obtain the image (and label) when needed. @@ -350,14 +427,25 @@ class LazyDataset(torch.utils.data.Dataset): * "guess" : Select a caching strategy via heuristic. """ + _supports_direct_batches = True + def __init__( # noqa: D107 self, func: Callable[[Any], torch.Tensor | tuple[torch.Tensor, ...] | list[torch.Tensor]], items: Sequence[Sequence], cache: str | int | CACHE_MODE | None = None, + *, + cache_workers: int | None = None, + pin_batches: bool = False, ): + if cache_workers is not None and (isinstance(cache_workers, bool) or not isinstance(cache_workers, int) or cache_workers < 0): + raise ValueError("cache_workers must be a nonnegative integer or None.") + self._cache_workers = cache_workers + self._pin_batches = pin_batches self.func = func self.items = tuple(np.asarray(seq, dtype=_infer_numeric_dtype(seq)) if len(seq) > 0 else np.empty((0,)) for seq in items) + if self.items and any(len(seq) != len(self.items[0]) for seq in self.items): + raise ValueError("Dataset input sequences must have equal lengths.") self._init_cache(CACHE_MODE(cache)) @staticmethod @@ -410,7 +498,7 @@ def _cache_ram(self, desc: str = "Writing to CPU RAM cache..."): self._ram_was_single_tensor = False templates = [e.new_empty(e.shape) for e in first_item_processed] else: - raise TypeError(f"The provided function must return a tensor ora tuple/list of tensors, but got {type(first_item_processed)}") + raise TypeError(f"The provided function must return a tensor or a tuple/list of tensors, but got {type(first_item_processed)}") stacked_tensors = [ torch.empty( @@ -422,85 +510,106 @@ def _cache_ram(self, desc: str = "Writing to CPU RAM cache..."): for template in templates ] - max_workers = _default_worker_count(128, reserve=2, minimum=1) - batch_size = min(256, 4 * max_workers) - fetch_pool = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="fetcher") - fetched_queue: Queue[tuple[int, torch.Tensor | Sequence[torch.Tensor] | Exception]] = Queue(max(32, batch_size * 4)) - insert_buffer: dict[int, torch.Tensor | Sequence[torch.Tensor]] = dict() - insert_queue: Queue[tuple[int, torch.Tensor | Sequence[torch.Tensor]]] = Queue() - - def _fetch_one(idx_item): - idx, item = idx_item - try: - data = self.func(item) - fetched_queue.put((idx, data)) - except Exception as e: - fetched_queue.put((idx, e)) - - def _contiguous_write(idx: list[int], data: list[torch.Tensor] | list[list[torch.Tensor]]) -> None: - """Write data to indexes. - - Args: - idx: A list of contigous increasing indices for each corresponding torch.Tensor (element) in `data`. - data: A list of torch.Tensor with the same length as `idx`. - """ - if len(idx) == 0: + max_workers = _default_worker_count(16) if self._cache_workers is None else self._cache_workers + max_workers = min(max_workers, len(self) - 1) + batch_size = min(64, max(1, 4 * max_workers)) + + def processed_items(): + # Reuse the shape probe: every reader/hook runs exactly once. + yield first_item_processed + items = iter(zip(*self.items)) + next(items) + if max_workers == 0: + yield from map(self.func, items) return - slc = slice(idx[0], idx[-1] + 1) - # Insert data into slice along first dimension in dst (in-place) - if self._ram_was_single_tensor: - assert not data or isinstance(data[0], torch.Tensor) - data = cast(list[torch.Tensor], data) - torch.stack(data, out=stacked_tensors[0][slc]) - else: - for i, elements in enumerate(zip(*data)): - torch.stack(elements, out=stacked_tensors[i][slc]) - - def _write(): - end_idx = len(self) - 1 - with TQDM(range(len(self)), desc=desc, leave=False) as pbar: - batch = ([], []) - while True: - idx, data = insert_queue.get() - batch[0].append(idx) - batch[1].append(data) - if len(batch[0]) >= batch_size: - _contiguous_write(*batch) - batch = ([], []) - pbar.update() - if idx == end_idx: + pool = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="mini-trainer-cache") + pending = deque() + try: + # Bound both submitted work and decoded samples even when an + # early reader or the cache writer is slower than later reads. + for _ in range(2 * max_workers): + item = next(items, None) + if item is None: break - _contiguous_write(*batch) - - write_thread = Thread(target=_write, daemon=True) - write_thread.start() - - try: - fetch_pool.map(_fetch_one, enumerate(zip(*self.items))) - - nxt_idx = 0 - for _ in range(len(self)): - idx, data = fetched_queue.get() - if isinstance(data, Exception): - raise data - if idx == nxt_idx: - insert_queue.put((idx, data)) - nxt_idx += 1 - while nxt_idx in insert_buffer: - insert_queue.put((nxt_idx, insert_buffer.pop(nxt_idx))) - nxt_idx += 1 + pending.append(pool.submit(self.func, item)) + while pending: + yield pending.popleft().result() + item = next(items, None) + if item is not None: + pending.append(pool.submit(self.func, item)) + finally: + pool.shutdown(wait=True, cancel_futures=True) + + # Writes happen here so shape/type errors reach the caller. Closing the + # generator also shuts down readers when stacking a batch fails. + with closing(processed_items()) as records, TQDM(total=len(self), desc=desc, leave=False) as pbar: + offset = 0 + for batch in batched(records, batch_size): + for index, record in enumerate(batch, offset): + values = (record,) if self._ram_was_single_tensor else record + if not isinstance(values, (tuple, list)) or len(values) != len(templates): + raise ValueError(f"Cached sample {index} has an inconsistent tensor structure.") + for value, template in zip(values, templates, strict=True): + if not isinstance(value, torch.Tensor) or value.shape != template.shape: + raise ValueError(f"Cached sample {index} has an inconsistent tensor shape.") + destination = slice(offset, offset + len(batch)) + if self._ram_was_single_tensor: + torch.stack(batch, out=stacked_tensors[0][destination]) else: - insert_buffer[idx] = data - - write_thread.join() - finally: - fetch_pool.shutdown(wait=False) + for target, values in zip(stacked_tensors, zip(*batch, strict=True), strict=True): + torch.stack(values, out=target[destination]) + offset += len(batch) + pbar.update(len(batch)) self._ram_cache = torch.utils.data.TensorDataset(*[t for t in stacked_tensors]) def __len__(self): return len(self.items[0]) + def __getitems__(self, indices): + # PyTorch's batched fetch protocol: one gather for cached tensors, or + # one stacking pass for decoded samples. Keep scalar indexing unchanged. + if self._cache_mode in (CACHE_MODE.CPU, CACHE_MODE.CUDA): + tensors = self._ram_cache.tensors + index = torch.as_tensor(indices, dtype=torch.long, device=tensors[0].device) + index = torch.where(index < 0, index + len(self), index) + # index_select copies whole rows; generic advanced indexing is much + # slower for uint8 image batches on CPU. Results own their storage. + pin = self._pin_batches and tensors[0].device.type == "cpu" and torch.utils.data.get_worker_info() is None + if pin: + # Gather directly into the final pinned batch. DataLoader's + # pinning pass then returns the same allocation, without a + # second full image copy. Never initialize CUDA in workers. + data = tuple( + torch.index_select( + tensor, + 0, + index, + out=torch.empty((len(index), *tensor.shape[1:]), dtype=tensor.dtype, device=tensor.device, pin_memory=True), + ) + for tensor in tensors + ) + else: + worker = isinstance(indices, _DirectBatchIndices) and torch.utils.data.get_worker_info() is not None + data = tuple( + torch.index_select( + tensor, + 0, + index, + out=_shared_batch_buffer(tensor, (len(index), *tensor.shape[1:])) + if worker and tensor.device.type == "cpu" and not tensor.requires_grad + else None, + ) + for tensor in tensors + ) + data = data[0] if self._ram_was_single_tensor else data + else: + data = self[indices] + # Ordinary external batched fetches still return actual sample lists, + # including direct torch.stack compatibility. Our sampler marks only + # batches whose collator can consume the stacked storage directly. + return _FetchedBatch(data, sample_views=not isinstance(indices, _DirectBatchIndices)) + def __getitem__(self, index): match self._cache_mode: case CACHE_MODE.NONE: diff --git a/mini_trainer/data/loader.py b/mini_trainer/data/loader.py index 806638fe..5ecce458 100644 --- a/mini_trainer/data/loader.py +++ b/mini_trainer/data/loader.py @@ -2,16 +2,19 @@ import numpy as np import torch -from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler +from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler, default_collate from torch.utils.data.distributed import DistributedSampler from mini_trainer import get_logger from mini_trainer.utils import is_dist_avail_and_initialized +from ._prefetch import CUDAPrefetchLoader from ._workers import _default_worker_count from .io import ( CACHE_MODE, LazyDataset, + _DirectBatchIndices, + _FetchedBatch, guess_cache_mode, make_read_and_resize_fn, ) @@ -67,8 +70,32 @@ def __call__(self, x: str) -> torch.Tensor: return self.hook(self.reader(x)) +class _DirectBatchSampler(BatchSampler): + """Retain normal sampling while requesting stacked batches from LazyDataset.""" + + def __iter__(self): + for indices in super().__iter__(): + yield _DirectBatchIndices(indices) + + +def _collate_batch(samples): + if isinstance(samples, _FetchedBatch): + # Match default_collate's tuple-to-list convention for (image, label). + return list(samples.data) if isinstance(samples.data, tuple) else samples.data + return default_collate(samples) + + def get_dataloader( # noqa: D103 - dataset: torch.utils.data.Dataset, mode: str, batch_size: int, num_workers: int, pin_memory: bool, device: torch.device + dataset: torch.utils.data.Dataset, + mode: str, + batch_size: int, + num_workers: int, + pin_memory: bool, + device: torch.device, + *, + cuda_prefetch: bool = False, + prefetch_factor: int | None = None, + multiprocessing_context: str | None = None, ): assert isinstance(mode, str) if mode.strip().lower() == "train": @@ -84,15 +111,24 @@ def get_dataloader( # noqa: D103 base_sampler = RandomSampler(dataset) if shuffle else SequentialSampler(dataset) # type: ignore mp_context = None - sampler = BatchSampler(base_sampler, batch_size=batch_size, drop_last=drop_last) + if num_workers > 0 and multiprocessing_context is not None: + mp_context = multiprocessing_context + + sampler_cls = _DirectBatchSampler if getattr(dataset, "_supports_direct_batches", False) else BatchSampler + sampler = sampler_cls(base_sampler, batch_size=batch_size, drop_last=drop_last) - return DataLoader( + loader_cls = CUDAPrefetchLoader if cuda_prefetch else DataLoader + transfer_kwargs = {"device": device} if cuda_prefetch else {} + return loader_cls( dataset, + **transfer_kwargs, batch_sampler=sampler, + collate_fn=_collate_batch, num_workers=num_workers, pin_memory=pin_memory, persistent_workers=num_workers > 0, multiprocessing_context=mp_context, + prefetch_factor=prefetch_factor if num_workers > 0 else None, ) @@ -107,12 +143,18 @@ def get_dataset_dataloader( # noqa: D103 device: torch.device | str = torch.device("cpu"), dtype: torch.dtype = torch.float32, cache: CACHE_MODE | str | int | None = None, + cache_workers: int | None = None, multilabel: bool = False, + cuda_prefetch: bool = False, + prefetch_factor: int | None = None, + multiprocessing_context: str | None = None, hook: Callable[[torch.Tensor], torch.Tensor] | None = None, ): resize_size = _normalize_resize_size(resize_size, error_suffix=".") if isinstance(device, str): device = torch.device(device) + if cuda_prefetch and device.type != "cuda": + raise ValueError("CUDA batch prefetch requires a CUDA target device.") if len(metadata) != len(modes): raise ValueError(f"Number of supplied datasets: {len(metadata)} and modes: {len(modes)} do not match!") @@ -131,21 +173,42 @@ def get_dataset_dataloader( # noqa: D103 proc_path_label = PathLabelProcessor(reader, hook, multilabel) - datasets = [] - for mode, data in zip(modes, metadata): - if mode.strip().lower() == "train" and resample: - raise NotImplementedError("Resampling is currently not supported.") - dset = LazyDataset(func=proc_path_label, items=(data["path"], data["class"]), cache=cache) - datasets.append(dset) - if cache is CACHE_MODE.CUDA: # When the entire dataset is preloaded there is no need to use multiprocessing for dataloading num_workers = 0 elif num_workers is None: num_workers = _default_worker_count(16) - pin_memory = cache not in [CACHE_MODE.CUDA, CACHE_MODE.CPU] - loaders = [get_dataloader(dataset, mode, batch_size, num_workers, pin_memory, device) for mode, dataset in zip(modes, datasets)] + datasets = [] + for mode, data in zip(modes, metadata): + if mode.strip().lower() == "train" and resample: + raise NotImplementedError("Resampling is currently not supported.") + dset = LazyDataset( + func=proc_path_label, + items=(data["path"], data["class"]), + cache=cache, + cache_workers=cache_workers, + pin_batches=device.type == "cuda" and cache is CACHE_MODE.CPU and num_workers == 0, + ) + datasets.append(dset) + + # Main-process CPU cache gathers are already pinned. Decoded and worker + # batches are pinned by DataLoader before asynchronous H2D. + pin_memory = device.type == "cuda" and cache is not CACHE_MODE.CUDA + loaders = [ + get_dataloader( + dataset, + mode, + batch_size, + num_workers, + pin_memory, + device, + cuda_prefetch=cuda_prefetch and cache is not CACHE_MODE.CUDA, + prefetch_factor=prefetch_factor, + multiprocessing_context=multiprocessing_context, + ) + for mode, dataset in zip(modes, datasets) + ] return datasets, loaders @@ -159,11 +222,16 @@ def get_inference_dataloader( # noqa: D103 device: torch.device | str = torch.device("cpu"), dtype: torch.dtype = torch.float32, hook: Callable[[torch.Tensor], torch.Tensor] | None = None, + prefetch_factor: int | None = None, + multiprocessing_context: str | None = None, + cuda_prefetch: bool = False, **kwargs, ): resize_size = _normalize_resize_size(resize_size) if isinstance(device, str): device = torch.device(device) + if cuda_prefetch and device.type != "cuda": + raise ValueError("CUDA batch prefetch requires a CUDA target device.") if subsample is not None and subsample > 1: images = images[::subsample] @@ -177,6 +245,16 @@ def get_inference_dataloader( # noqa: D103 if num_workers is None: num_workers = _default_worker_count(32) - loader = get_dataloader(dataset, "test", batch_size, num_workers, False, device) + loader = get_dataloader( + dataset, + "test", + batch_size, + num_workers, + device.type == "cuda", + device, + cuda_prefetch=cuda_prefetch, + prefetch_factor=prefetch_factor, + multiprocessing_context=multiprocessing_context, + ) return dataset, loader diff --git a/mini_trainer/data/metadata.py b/mini_trainer/data/metadata.py index a6c2f426..12d2c014 100644 --- a/mini_trainer/data/metadata.py +++ b/mini_trainer/data/metadata.py @@ -241,29 +241,93 @@ def partition_class_samples( return split_samples, violations -def auto_find_images(src: str, **kwargs) -> tuple[list[int] | list[list[int]], list[str]]: - """Find images in source and possibly create training metadata.""" - metadata = labels = images = None +def _inference_folder_labels(folders: list[str], cls2idx: dict, labels) -> OrderedDict: + """Resolve input folders independently of the model's prediction vocabulary.""" + levels = len(cls2idx) + known = labels if isinstance(labels, dict) else {str(value[0]): value for value in (labels or [])} + resolved = OrderedDict() + missing = [] + for name in folders: + value = known.get(name) + if isinstance(value, (list, tuple)) and len(value) == levels: + resolved[name] = tuple(map(str, value)) + else: + missing.append(name) + if missing: + if not is_taxonomical_cls2idx(cls2idx): + raise ValueError(f"Missing hierarchical labels for input folders: {missing}") + # An integer is an inclusive deepest-rank index in create_taxonomy, + # not a level count: 3 would include order as a fourth level. + fetched = labels_from_taxonomy(create_taxonomy(missing, list(range(levels)))) + for name in missing: + value = fetched.get(name) + if value is None or len(value) != levels: + raise ValueError(f"Could not resolve {levels} hierarchy levels for input folder {name!r}; received {value!r}") + # Folder IDs are supplied ground truth; do not silently canonicalize + # unseen/synonymous species into the model's vocabulary. + resolved[name] = (name if name.isdigit() else str(value[0]), *map(str, value[1:])) + return OrderedDict((name, resolved[name]) for name in folders) + + +def auto_find_images( + src: str, + *, + cls2idx: dict | None = None, + labels: dict | list | None = None, + **kwargs, +) -> tuple[list[str | tuple[str, ...] | list[str]], list[str]]: + """Return raw ground truth and image paths for prediction (never class indices). + + Parquet inputs retain their supplied test split; ``cls2idx``, ``labels`` and + remaining metadata options are forwarded to the Parquet adapter. Other files + are returned as single unlabelled inputs, without validating image contents. + + Directories are scanned recursively in sorted path order. If every image is + below an immediate child directory, that directory names its class. Root-level + images instead make the entire input unlabelled. Non-image files and empty + folders do not affect this decision. Recognized split/class/image layouts + select only the supplied test split. Discovery does not partition or shuffle + samples, create an index, or restrict coverage to the model vocabulary. + + ``labels`` optionally maps folder names to ground truth (or lists labels keyed + by their first element for hierarchical labels). Hierarchical ``cls2idx`` + declares the expected number of levels. Missing hierarchical mappings retain + the existing GBIF/cache lookup for taxonomical vocabularies, with explicit + failure if resolution fails. This can involve network access. Numeric folder + IDs are preserved for newly resolved species. Other kwargs do not affect + directory discovery. Unlabelled inputs return ``([], images)``. + """ if os.path.isfile(src): if src.endswith(".parquet"): - metadata = get_metadata_from_parquet(src, **kwargs) - else: - images = [src] - elif os.path.isdir(src): - contains_only_dirs = all([os.path.isdir(os.path.join(src, p)) for p in os.listdir(src)]) - if contains_only_dirs: - metadata = create_metadata(src, **{**kwargs, **{"train_proportion": 0, "val_proportion": 0}}) - else: - images = find_images(src) - else: + metadata = get_metadata_from_parquet(src, cls2idx=cls2idx, labels=labels, **kwargs) + samples = [(c, p) for c, p, split in zip(metadata["label"], metadata["path"], metadata["split"]) if split == "test"] + return [c for c, _ in samples], [p for _, p in samples] + return [], [src] + if not os.path.isdir(src): raise ValueError(f"Image source must be a file (image or gbifxdl parquet) or directory with images, not {src}.") - if metadata is not None: - images = [p for p, s in zip(metadata["path"], metadata["split"]) if s == "test"] - labels = [c for c, s in zip(metadata["label"], metadata["split"]) if s == "test"] - assert images is not None - if labels is None: - labels = [] - return labels, images + + images = find_images(src) + folders = [Path(os.path.relpath(path, src)).parts for path in images] + if not images or any(len(parts) == 1 for parts in folders): + return [], images + # Preserve supplied directory splits without constructing training metadata. + if all(parts[0].lower() in SPLIT_DIR_MAP and len(parts) >= 3 for parts in folders): + selected = [(path, parts[1:]) for path, parts in zip(images, folders) if SPLIT_DIR_MAP[parts[0].lower()] == "test"] + images = [path for path, _ in selected] + folders = [parts for _, parts in selected] + if not images: + return [], [] + names = [parts[0] for parts in folders] + if isinstance(cls2idx, dict) and isinstance(cls2idx.get("0"), dict): + mapping = _inference_folder_labels(sorted(set(names)), cls2idx, labels) + else: + mapping = ( + labels + if isinstance(labels, dict) + else {str(value[0] if isinstance(value, (list, tuple)) else value): value for value in (labels or [])} + ) + ground_truth = [mapping.get(name, name) for name in names] + return [tuple(value) if isinstance(value, list) else value for value in ground_truth], images def label_to_class_idx( diff --git a/mini_trainer/export.py b/mini_trainer/export.py index 7d7fa303..535f9ed4 100644 --- a/mini_trainer/export.py +++ b/mini_trainer/export.py @@ -10,6 +10,7 @@ from mini_trainer.modeling import Classifier, classification_module from mini_trainer.modeling.architectures.load import get_dynamic_model, resolve_backbone_getter from mini_trainer.modeling.onnx import export_onnx +from mini_trainer.modeling.quantized_training import load_training_weights, materialize_quantized_training_state def main( @@ -20,14 +21,18 @@ def main( model_args: dict | None = None, dynamic_batch: bool = True, batch_size: int = 2, + reference_device: str = "cpu", + materialize_int8_training: bool = False, ): if batch_size < 1: raise ValueError("batch_size must be positive.") with Path(weights).open("rb") as handle: checkpoint_hash = hashlib.file_digest(handle, "sha256").hexdigest() - handle.seek(0) - state = torch.load(handle, map_location="cpu", weights_only=True) + state = load_training_weights(weights, map_location="cpu") state = state.get("model", state) + conversion = None + if materialize_int8_training: + state, conversion = materialize_quantized_training_state(state, dtype=torch.float32) metadata = Classifier.extract_metadata(state) model_type = metadata.get("backbone_class") if not model_type: @@ -45,7 +50,21 @@ def main( if not input_shape or any(size < 1 for size in input_shape): raise ValueError("input_shape must contain positive non-batch dimensions.") sample = torch.zeros(batch_size, *input_shape) - return export_onnx(model, sample, output, preprocessing=preprocessing, dynamic_batch=dynamic_batch, checkpoint_sha256=checkpoint_hash) + bundle = export_onnx( + model, + sample, + output, + preprocessing=preprocessing, + dynamic_batch=dynamic_batch, + checkpoint_sha256=checkpoint_hash, + reference_device=reference_device, + ) + if conversion is not None: + manifest_path = bundle / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["source"]["quantized_training_materialization"] = conversion + manifest_path.write_text(json.dumps(manifest, indent=2, allow_nan=False) + "\n") + return bundle def run(): @@ -57,6 +76,15 @@ def run(): parser.add_argument("--model-args", type=json.loads, help="JSON constructor arguments for the existing architecture loader.") parser.add_argument("--static-batch", action="store_false", dest="dynamic_batch", help="Export a fixed batch size.") parser.add_argument("--batch-size", type=int, default=2, help="Example/static batch size (default: 2).") + parser.add_argument("--reference-device", default="cpu", help="PyTorch parity device; native INT8 training requires cuda.") + parser.add_argument( + "--materialize-int8-training", + action="store_true", + help=( + "Explicit floating export for subsequent calibration; " + "removes native dynamic activation quantization and requires quality validation." + ), + ) args = vars(parser.parse_args()) if args["preprocessing"] is not None: args["preprocessing"] = json.loads(args["preprocessing"].read_text()) diff --git a/mini_trainer/integrations/gbif.py b/mini_trainer/integrations/gbif.py index d3fed51f..5864d856 100644 --- a/mini_trainer/integrations/gbif.py +++ b/mini_trainer/integrations/gbif.py @@ -28,8 +28,14 @@ def get_cache(): return _CACHE -def retrive_request(req: str) -> Any: - """Retrieve a composed HTTPS request.""" +def retrive_request(req: str, *, timeout: float | None = None) -> Any: + """Retrieve and cache an HTTPS response. + + ``timeout`` bounds blocking socket operations, not an entire taxonomy lookup. + Omit it to use urllib's process-wide socket default. Callers that need a total + batch deadline must enforce that at the orchestration boundary; this transport + helper cannot determine an appropriate workload budget. + """ if not req.startswith("https://"): raise NotImplementedError("Only HTTPS APIs are currently supported.") cache = get_cache() @@ -38,7 +44,7 @@ def retrive_request(req: str) -> Any: if cached_result is not None: return cached_result - with urlopen(req) as resp: + with urlopen(req, **({"timeout": timeout} if timeout is not None else {})) as resp: if resp.status != 200: raise RuntimeError(f"Unable to resolve request, received status {resp.status} from {req}.") data = json.load(resp) diff --git a/mini_trainer/logging/confusion.py b/mini_trainer/logging/confusion.py new file mode 100644 index 00000000..e56a95e6 --- /dev/null +++ b/mini_trainer/logging/confusion.py @@ -0,0 +1,174 @@ +"""Whole-matrix confusion diagnostics: bounded reduction, images and exact data. + +Dashboard overviews and full-resolution local artifacts share a fixed log scale. +No class filtering or reordering occurs here. Model output indices identify axes. +""" + +import json +import math +from pathlib import Path + +import matplotlib as mpl +import numpy as np +import torch +import torch.distributed as dist +from matplotlib.colors import LogNorm +from PIL import Image, ImageDraw + +from mini_trainer.visualization.plot import _generate_colorbar_rgb_array, _get_colorbar_ticks_and_labels + +PREVIEW_SIZE = 1536 +PREVIEW_HEADER_HEIGHT = 24 +PROBABILITY_MIN = 1e-6 +SOFT_COLOR_LEVELS = 128 +CHUNK_ELEMENTS = 262144 + + +def reduce_matrix(matrix, *, copy=True): + """Sum rank-local CPU matrices onto rank zero without a full GPU copy. + + Each call is collective; all ranks must supply the same shape and dtype. + Soft buffers are copied so repeated reporting cannot double counts. Fresh + hard-count buffers may pass copy=False to avoid another dense allocation. + """ + if not dist.is_available() or not dist.is_initialized(): + return matrix + device = torch.device("cuda", torch.cuda.current_device()) if dist.get_backend() == "nccl" else torch.device("cpu") + result = (np.empty_like(matrix) if copy else matrix) if dist.get_rank() == 0 else None + rows = max(1, CHUNK_ELEMENTS // max(1, matrix.shape[1])) + for start in range(0, matrix.shape[0], rows): + chunk = torch.tensor(matrix[start : start + rows], device=device) + dist.reduce(chunk, dst=0) + if result is not None: + result[start : start + rows] = chunk.cpu().numpy() + return result + + +def hard_counts(labels, predictions, n_classes): + """Retain all model output columns, including prediction-only classes.""" + labels, predictions = np.asarray(labels, dtype=np.int64), np.asarray(predictions, dtype=np.int64) + if labels.shape != predictions.shape: + raise ValueError("Labels and predictions must have the same shape") + if labels.size and (min(labels.min(), predictions.min()) < 0 or max(labels.max(), predictions.max()) >= n_classes): + raise ValueError("Confusion class index outside model output range") + return np.bincount(n_classes * labels + predictions, minlength=n_classes * n_classes).reshape(n_classes, n_classes) + + +def _palette(soft): + cmap = mpl.colormaps["magma"].resampled(SOFT_COLOR_LEVELS) if soft else mpl.colormaps["magma"].copy() + cmap.set_bad("magenta") + cmap.set_under("black") + palette = np.zeros((256, 3), dtype=np.uint8) + palette[1 : SOFT_COLOR_LEVELS + 1] = cmap(np.linspace(0, 1, SOFT_COLOR_LEVELS), bytes=True)[:, :3] + palette[SOFT_COLOR_LEVELS + 1] = (255, 0, 255) + return cmap, palette + + +def _colors(values, soft, cmap): + """Positive probabilities below the floor stay distinct from exact zero.""" + invalid = ~np.isfinite(values) | (values < 0) + zero = values == 0 + log_values = np.log10(np.clip(np.where(invalid, 0, values), PROBABILITY_MIN, 1)) + normalized = (log_values - math.log10(PROBABILITY_MIN)) / -math.log10(PROBABILITY_MIN) + if soft: + indices = np.minimum((normalized * SOFT_COLOR_LEVELS).astype(np.uint8), SOFT_COLOR_LEVELS - 1) + 1 + indices[zero] = 0 + indices[invalid] = SOFT_COLOR_LEVELS + 1 + return indices + rgb = cmap(normalized, bytes=True)[:, :, :3].copy() + rgb[zero] = 0 + rgb[invalid] = (255, 0, 255) + return rgb + + +def confusion_report(matrix, *, soft, directory=None): + """Return a bounded whole-matrix preview; optionally save native-resolution data. + + Large previews show the arithmetic mean of row-normalized cell probabilities + in each block. They are overviews, not aggregated/renormalized class matrices. + Exact hard counts are sparse NPZ; soft sums are uncompressed NPY to avoid + expensive compression of high-entropy floats during training. + """ + matrix = np.asarray(matrix) + if matrix.ndim != 2 or not matrix.shape[0] or matrix.shape[0] != matrix.shape[1]: + raise ValueError("Confusion report requires a nonempty square matrix") + n = len(matrix) + support = matrix.sum(axis=1, dtype=np.float64 if soft else np.int64) + block = max(1, math.ceil(n / PREVIEW_SIZE)) + edges = np.arange(0, n, block) + widths = np.minimum(block, n - edges) + preview = np.zeros((len(edges), len(edges)), dtype=np.float64) + cmap, palette = _palette(soft) + full = None + if directory is not None: + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + full = np.empty((n, n) if soft else (n, n, 3), dtype=np.uint8) + if soft: + np.save(directory / "probability_sums.npy", matrix, allow_pickle=False) + else: + rows, columns = np.nonzero(matrix) + np.savez_compressed(directory / "counts.npz", rows=rows, columns=columns, counts=matrix[rows, columns], shape=matrix.shape) + np.save(directory / "row_support.npy", support, allow_pickle=False) + rows_per_chunk = max(block, (CHUNK_ELEMENTS // max(1, n) // block) * block) + invalid_cells = 0 + for start in range(0, n, rows_per_chunk): + raw = matrix[start : start + rows_per_chunk] + denominator = support[start : start + len(raw), None] + values = np.zeros(raw.shape, dtype=np.float64) + with np.errstate(invalid="ignore", divide="ignore"): + np.divide(raw, denominator, out=values, where=denominator != 0) + values[(raw < 0) | ~np.isfinite(raw)] = np.nan + invalid_cells += int(np.count_nonzero(~np.isfinite(values) | (values < 0))) + if full is not None: + full[start : start + len(raw)] = _colors(values, soft, cmap) + sums = np.add.reduceat(np.add.reduceat(values, edges, axis=1), np.arange(0, len(raw), block), axis=0) + lengths = np.minimum(block, len(raw) - np.arange(0, len(raw), block)) + preview[start // block : start // block + len(sums)] = sums / lengths[:, None] / widths[None, :] + + def image(values): + if soft: + result = Image.fromarray(values) + result.putpalette(palette.ravel().tolist()) + return result + return Image.fromarray(values) + + if directory is not None: + image(full).save(directory / "matrix.png", compress_level=3) + del full + metadata = { + "shape": [n, n], + "class_indices": list(range(n)), + "axes": "rows=true class; columns=predicted class; original model output index order", + "class_names": "Resolve indices using the saved model/config classifier metadata", + "normalization": "divide each raw row by its sum; empty rows remain zero", + "raw_values": "probability_sums.npy" if soft else "counts.npz", + "support": "row_support.npy", + "scope": "sum over reporting ranks (including any validation sampler padding)", + "probability_scale": [PROBABILITY_MIN, 1], + "positive_color_levels": SOFT_COLOR_LEVELS if soft else cmap.N, + "zero_color": "black", + "invalid_color": "magenta", + "invalid_normalized_cells": invalid_cells, + "preview": {"method": "arithmetic mean of cell probabilities", "block_shape": [block, block]}, + } + (directory / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + preview_image = image(_colors(preview, soft, cmap)).convert("RGB") + # Nearest-neighbor enlargement for small matrices only; never smooth cells. + if preview_image.width < 500: + factor = math.ceil(500 / preview_image.width) + preview_image = preview_image.resize((preview_image.width * factor, preview_image.height * factor), Image.Resampling.NEAREST) + norm = LogNorm(PROBABILITY_MIN, 1) + ticks, labels = _get_colorbar_ticks_and_labels(PROBABILITY_MIN, 1, 8, True) + colorbar = _generate_colorbar_rgb_array(norm, cmap, ticks, labels, preview_image.height, 10) + if directory is not None: + Image.fromarray(colorbar).save(directory / "colorbar.png", compress_level=3) + combined = Image.fromarray(np.hstack((np.asarray(preview_image), colorbar))) + captioned = Image.new("RGB", (combined.width, combined.height + PREVIEW_HEADER_HEIGHT), "white") + captioned.paste(combined, (0, PREVIEW_HEADER_HEIGHT)) + ImageDraw.Draw(captioned).text( + (8, 4), + f"{n} classes | rows=true, columns=predicted | block mean {block}x{block} | black=0; magenta=invalid", + fill="black", + ) + return np.asarray(captioned) diff --git a/mini_trainer/logging/core.py b/mini_trainer/logging/core.py index d9b23a41..50d55ae9 100644 --- a/mini_trainer/logging/core.py +++ b/mini_trainer/logging/core.py @@ -1,11 +1,14 @@ import math import os +import re +import shutil import time import warnings from abc import ABC, abstractmethod from collections import defaultdict, deque from collections.abc import Callable, Iterator, Sequence from itertools import chain, repeat +from pathlib import Path from tempfile import NamedTemporaryFile from threading import RLock from typing import Any, TextIO, TypeVar @@ -19,19 +22,21 @@ except ImportError: psutil = None import torch +import torch.distributed as dist from matplotlib import pyplot as plt from matplotlib.figure import Figure from torch import nn from mini_trainer import get_logger -from mini_trainer.training import raw_confusion_matrix from mini_trainer.utils import float_signif_decimal, get_rank, main_process_first, reduce_across_processes, write_csv_from_dict from mini_trainer.visualization import ( plot_class_distance_matrix, - plot_heatmap, plot_probabilistic_dendrogram, + save_dendrogram_svg, ) +from .confusion import confusion_report, hard_counts, reduce_matrix + def format_duration(sec: int | float, suffix="dhms"): """Format duration in seconds.""" @@ -1033,74 +1038,96 @@ def render_soft_confusion_matrix(self, labels: torch.Tensor, predictions: torch. self._soft_confusion_matrix[level] = cf def confusion_matrix(self): - lvl = None - figs: dict[str, np.ndarray] = dict() - while True: - # Check if there is one or multiple levels, and if so if the current level exists - counts: dict[str, list[int]] - if lvl is None: - counts = {"labels": [], "predictions": []} - lvl = 0 - if any([key not in self.heterogeneous_storage for key in counts]): - continue - else: - counts = {f"labels/lvl{lvl}": [], f"predictions/lvl{lvl}": []} - if any([key not in self.heterogeneous_storage for key in counts]): - break - - # Find label and prediction combinations (at the current level if applicable) - hits = 0 - for what in counts: - for cls_idxs, epoch, tp in reversed(self.heterogeneous_storage[what]): - if epoch != self._epoch: - break - ctp = tp.lower().strip() - if not (ctp.startswith("val") or ctp.startswith("eval")): - continue - hits += 1 - counts[what].extend(cls_idxs) - if hits == 0: - get_logger().warning(f"No labels or predictions found for {self._epoch}!") - - # Create confusion matrix from counts - cm = raw_confusion_matrix(*counts.values(), n_classes=None) - m = cm.sum(axis=1) > 0 - cm = cm[m][:, m] - if not bool(np.any(np.isfinite(cm))): - get_logger().warning(f"Confusion matrix has no valid values, produced from counts: {counts}") - - cm_lab = "Confusion matrix" - if lvl is not None: - cm_lab += f"/lvl{lvl}" - figs[cm_lab] = plot_heatmap(cm) - - lvl += 1 - - if len(self._soft_confusion_matrix) == 0: - pass - elif len(self._soft_confusion_matrix) == 1: - cm = list(self._soft_confusion_matrix.values())[0] - zm = cm.max(dim=1).values < (1 / cm.sum()) - cm = cm[~zm][:, ~zm] - cm_rs = cm.sum(dim=1, keepdim=True) - cm = cm / cm_rs - cm = cm.clamp(1 / cm_rs.sum().item(), 1) - figs["Soft confusion matrix"] = plot_heatmap(cm) - else: - for lvl, cm in self._soft_confusion_matrix.items(): - zm = cm.max(dim=1).values < (1 / cm.sum()) - cm = cm[~zm][:, ~zm] - cm_rs = cm.sum(dim=1, keepdim=True) - cm = cm / cm_rs - cm = cm.clamp(1 / cm_rs.sum().item(), 1) - figs[f"Soft confusion matrix/lvl{lvl}"] = plot_heatmap(cm) - return figs + """Collect global whole-matrix diagnostics; only rank zero renders.""" + started = time.monotonic() + specifications = {} + for level, matrix in self._soft_confusion_matrix.items(): + prefix = "labels" if level == 0 and "labels" in self.heterogeneous_storage else f"labels/lvl{level}" + specifications[level] = (matrix.shape[0], prefix, True) + for key in list(self.heterogeneous_storage): + if key == "labels" or key.startswith("labels/lvl"): + level = 0 if key == "labels" else int(key.split("lvl")[-1]) + if level not in specifications: + values = [] + for field in (key, key.replace("labels", "predictions")): + for indices, epoch, phase in self.heterogeneous_storage.get(field, []): + if epoch == self._epoch and phase.lower().startswith(("eval", "val")): + values.extend(indices) + specifications[level] = (max(self._n_classes or 0, max(values, default=-1) + 1), key, False) + if dist.is_available() and dist.is_initialized(): + # Small shape descriptors also cover ranks with no validation batches. + gathered = [None] * dist.get_world_size() + dist.all_gather_object(gathered, specifications) + for other in gathered: + for level, (size, prefix, has_soft) in other.items(): + existing = specifications.get(level, (0, prefix, False)) + specifications[level] = (max(size, existing[0]), prefix, has_soft or existing[2]) + figures = {} + for level, (size, prefix, has_soft) in sorted(specifications.items()): + if not size: + continue + counts = [] + for field in (prefix, prefix.replace("labels", "predictions")): + values = [] + for indices, epoch, phase in self.heterogeneous_storage.get(field, []): + if epoch == self._epoch and phase.lower().startswith(("eval", "val")): + values.extend(indices) + counts.append(values) + matrix = reduce_matrix(hard_counts(*counts, size), copy=False) + if get_rank() == 0: + get_logger().info(f"Rendering whole-matrix confusion diagnostics: level {level}, {size:,} classes") + name = f"Confusion matrix/lvl{level}" + directory = self._confusion_directory(name) + figures[name + "/overview_mean"] = confusion_report(matrix, soft=False, directory=directory) + del matrix + if not has_soft: + continue + local_soft = self._soft_confusion_matrix.get(level) + if local_soft is None: + local_soft = torch.zeros((size, size), dtype=torch.float32) + matrix = reduce_matrix(local_soft.detach().numpy()) + if get_rank() == 0: + name = f"Soft confusion matrix/lvl{level}" + directory = self._confusion_directory(name) + figures[name + "/overview_mean"] = confusion_report(matrix, soft=True, directory=directory) + del matrix + if figures: + get_logger().info(f"Confusion diagnostics generated in {time.monotonic() - started:.1f}s") + return figures + + def _confusion_directory(self, name): + if self.output_dir is None: + return None + epoch = "unknown" if self._epoch is None else f"{self._epoch + 1:04d}" + return Path(self.output_dir) / "figures" / f"epoch-{epoch}" / name.replace(" ", "_").replace("/", "_") def add_figure(self, name: str, figure: Figure | np.ndarray | torch.Tensor | str): - for logger in self.loggers: - logger.add_figure(name=name, figure=figure, epoch=self._epoch) - if isinstance(figure, Figure): - plt.close(figure) + try: + if self.output_dir is not None and get_rank() == 0: + epoch = "unknown" if self._epoch is None else f"{self._epoch + 1:04d}" + directory = Path(self.output_dir) / "figures" / f"epoch-{epoch}" + directory.mkdir(parents=True, exist_ok=True) + stem = re.sub(r"[^\w.-]+", "_", name).strip(".") or "figure" + if isinstance(figure, Figure): + figure.savefig(directory / f"{stem}.png", bbox_inches="tight") + elif isinstance(figure, str): + if "