From aca8021b59d1b2905f62288371ed96e692453857 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Sun, 16 Aug 2026 23:27:16 -0700 Subject: [PATCH 01/10] docs: correct inaccurate and stale documentation across the repo --- .github/workflows/ci.yml | 4 +- CONTRIBUTING.md | 35 ++-- README.md | 4 +- ROADMAP.md | 23 ++- SECURITY.md | 7 +- api/README.md | 12 +- clients/python/README.md | 7 +- clients/submitter/README.md | 29 +++- cmd/sqi-server/README.md | 20 ++- cmd/sqi-worker/README.md | 17 +- cmd/sqi-worker/main.go | 5 +- cmd/sqi-worker/root.go | 5 +- config/sqi-server.example.yaml | 13 +- config/sqi-worker.example.yaml | 50 +++--- deploy/README.md | 2 +- docs/README.md | 4 +- docs/api.md | 187 ++++++++++++++++++--- docs/architecture.md | 115 +++++++++---- docs/auth.md | 55 ++++-- docs/compute-locations.md | 2 +- docs/configuration.md | 74 +++++--- docs/contributing.md | 30 ++-- docs/cross-platform-validation.md | 25 +-- docs/dcc-submitters.md | 18 +- docs/development.md | 53 +++++- docs/index.md | 4 +- docs/observability.md | 20 ++- docs/openjd-conformance.md | 1 + docs/openjd-extensions.md | 1 + docs/openjd-extensions/expr.md | 27 ++- docs/openjd-extensions/path-translation.md | 1 + docs/openjd-extensions/sqi-chunk-bounds.md | 1 + docs/openjd-extensions/task-chunking.md | 1 + docs/openjd-submission.md | 31 +++- docs/operations.md | 79 +++++---- docs/preset-library.md | 41 +++-- docs/products.md | 100 +++++++---- docs/python-client.md | 70 +++++++- docs/quickstart.md | 2 +- docs/release-runbook.md | 16 +- docs/roadmap.md | 53 +++++- docs/storage-s3.md | 2 +- docs/web-accessibility.md | 6 +- docs/web-development.md | 8 +- docs/worker-capabilities.md | 34 ++-- docs/worker-configuration.md | 72 +++++--- docs/worker-docker.md | 1 + internal/README.md | 49 +++++- internal/api/openapi.yaml | 30 ++-- internal/config/config.go | 4 +- internal/metrics/metrics.go | 2 +- internal/openjd/exprlimits.go | 44 ++--- internal/product/sqipresets_test.go | 15 +- internal/store/statemachine.go | 7 +- pkg/README.md | 4 +- pkg/doc.go | 10 +- scripts/README.md | 13 +- test/README.md | 10 +- web/README.md | 4 +- 59 files changed, 1125 insertions(+), 434 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb51100..3aa1a334 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,7 +168,7 @@ jobs: # the minimum line-coverage threshold. # `make test-cover` enables -race via RACE=on (the Makefile default), # writes coverage.out, and exits non-zero if coverage < COVERAGE_MIN. - # COVERAGE_MIN defaults to 60 in the Makefile — that is the single + # COVERAGE_MIN defaults to 70 in the Makefile — that is the single # source of truth; raise it there as coverage grows. - name: Test (race detector + coverage gate) run: make test-cover @@ -226,7 +226,7 @@ jobs: run: npm run typecheck # Vitest run with V8 coverage; exits non-zero if thresholds are not met. - # The threshold is defined in vite.config.ts (60 % — matches Go's minimum). + # The thresholds are defined in vite.config.ts (85 % lines / 82 % functions). - name: Test with coverage gate (Vitest) working-directory: web run: npm run test:coverage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 178180cf..6f745344 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,18 +17,21 @@ We maintain a [roadmap and architecture guide](ROADMAP.md) organized by developm git clone https://github.com/Uberware/sqi.git cd sqi -# Go development (server and workers) -cd cmd/sqi-server -go mod download -go build +# Install the git hooks (gofumpt, goimports, go vet, golangci-lint, and the +# Conventional Commits check). Requires gofumpt, goimports and golangci-lint +# on your PATH — see docs/development.md for install commands. +make hooks -# Python development (client SDK and DCC submitters) -cd clients/python -pip install -e . +# Go development (server and workers) — builds the web UI bundle first and +# writes both binaries into ./bin/ +make build + +# Python development (client SDK) +make py-install # Web UI development (TypeScript + React) cd web -npm install +npm ci npm run dev ``` @@ -43,13 +46,17 @@ npm run dev - **Go**: Follow [Effective Go](https://golang.org/doc/effective_go). Run `make fmt` (gofumpt + goimports) and `make lint` (golangci-lint) before submitting - **Python**: Follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Use type hints for all function signatures. Format and lint with ruff: `ruff format && ruff check --fix` — this handles style consistency automatically - **TypeScript/React**: Use ESLint and Prettier with the project configuration. Functional components and hooks preferred -- **Commit messages**: Be clear and specific. Reference issue numbers where relevant (e.g., "Fix scheduler race condition in Phase 1 (#42)") +- **SPDX header**: every source file (Go, TypeScript, YAML, Python, shell) must carry `SPDX-License-Identifier: AGPL-3.0-or-later` before any `package`/`import`/module declaration, in the file's own comment syntax. See [`docs/spdx-header.md`](docs/spdx-header.md) for the exact template. +- **Commit messages**: [Conventional Commits](https://www.conventionalcommits.org) format — `type(scope)?: description` — is **enforced** by the `commit-msg` git hook installed by `make hooks`. Valid types: `feat fix docs style refactor test chore build ci perf revert`. Reference issue numbers in the description where relevant (e.g. `fix(scheduler): correct heartbeat timeout calculation (#42)`). `CHANGELOG.md` is generated from these messages by git-cliff, so a non-conforming commit never reaches the changelog. **Testing:** - Unit tests are required for code changes. The enforced coverage floor is 70% (`COVERAGE_MIN` in the Makefile); aim higher on new code -- Integration tests are encouraged for complex features +- Integration tests are encouraged for complex features. They live in `test/integration/`; those behind the `integration` build tag need `make test-integration`, because they depend on something the default suite should not require (a built binary, or an external service) - Run `make test` (or `make ci`) before submitting a PR — do not run bare `go test ./...` from the repo root, since `web/node_modules/` contains third-party Go files the Makefile filters out +- **Changing LDAP code?** Run `make test-ldap`. It drives the login path against a real OpenLDAP server in a throwaway container, which is the only thing that catches a mistake in how sqi talks to a directory *on the wire* +- **Changing OIDC/SSO code?** Run `make test-oidc`. It drives the whole browser flow against a real Keycloak in a throwaway container, which is the only thing that catches what a real provider *omits* — most importantly a missing group claim, which validates fine and silently drops every user to `default_role` +- Both need Docker and **skip** without it — and a skip verifies nothing, so confirm the tests actually ran rather than trusting the exit code. Details in [`docs/development.md`](docs/development.md#testing-against-a-real-directory-or-identity-provider) **Submitting a PR:** @@ -92,8 +99,8 @@ Conventions: - Mock at the network boundary (the `apiFetch`/query layer), not at the component internals, so tests exercise real component wiring. - New components and hooks ship with tests; coverage is enforced against the - threshold in `web/vite.config.ts`. See the existing `DataTable.test.tsx`, - `StatusBadge.test.tsx`, and the `src/api` / `src/hooks` tests for the patterns. + threshold in `web/vite.config.ts`. See the existing `StatusBadge.test.tsx`, + `Pagination.test.tsx`, and the `src/api` / `src/hooks` tests for the patterns. **API client pattern.** All server access goes through `src/api/` — never call `fetch` directly from a component. @@ -207,8 +214,8 @@ The [ROADMAP.md](ROADMAP.md) document outlines development phases. Current prior - **Phase 1** (v0.1 — released): Core scheduler, pull-based workers, basic web UI, OpenJD execution - **Phase 2** (v0.2 — released): Product system, preset library, DCC submitters -- **Phase 3** (next): Auth (LDAP, OAuth2), multi-user role model -- **Phase 4** (planned): Production hardening, PostgreSQL, HA, auto-scaling +- **Phase 3** (complete, unreleased): Auth (local accounts, API keys, RBAC, LDAP/AD, OAuth2/OIDC SSO), multi-user role model, and run-as-user task isolation +- **Phase 4** (next): Production hardening, PostgreSQL, HA, auto-scaling Code contributions aligned with the current phase are most likely to be accepted quickly. Contributions targeting later phases are welcome but may take longer to review if they require design discussion. diff --git a/README.md b/README.md index ee60cda9..0a4740a2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ 📖 **[Documentation](https://uberware.github.io/sqi/)** — quickstart, architecture, configuration, and full reference. -> **Status:** v0.2.0 (Phase 2) released. Builds on the Phase 1 core with products and presets as an authoring layer over OpenJD, a community preset-library integration, product-driven submission, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** (opt-in auth & multi-user) — identity complete, task isolation in progress. Contributions, feedback, and discussion welcome. +> **Status:** v0.2.0 (Phase 2) released. Builds on the Phase 1 core with products and presets as an authoring layer over OpenJD, a community preset-library integration, product-driven submission, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** (opt-in auth & multi-user) — local accounts, API keys, RBAC, job-owner identity, LDAP/AD, OAuth2/OIDC SSO, and run-as-user task isolation — is complete and merged on `main`, unreleased. **Phase 4** (production hardening) is next. Contributions, feedback, and discussion welcome. --- @@ -24,7 +24,7 @@ The render farm management space is in an awkward moment. Legacy on-premises sys **Not tied to any cloud provider.** Workers run on Linux, macOS, and Windows — bare metal, VMs, or containers. Cloud compute locations are supported across AWS, GCP, Azure, and any provider that can run a container or a binary. Your control plane runs where you want it. -**OpenJD compatible.** `sqi` adopts the [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications) format as its native job execution layer — not an import step or a partial translation, but the format `sqi` jobs are authored and validated in directly. This is a real standard designed for portability, not a proprietary format. Conformance is measured, not asserted: `sqi` runs the official OpenJD conformance test suite on every CI build and publishes the current pass rate and known gaps at [`docs/openjd-conformance.md`](docs/openjd-conformance.md). One caveat is by design rather than a gap: a template that opts into an OpenJD extension `sqi` does not implement — such as `EXPR` — is rejected outright, rather than being accepted and misinterpreted. +**OpenJD compatible.** `sqi` adopts the [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications) format as its native job execution layer — not an import step or a partial translation, but the format `sqi` jobs are authored and validated in directly. This is a real standard designed for portability, not a proprietary format. Conformance is measured, not asserted: `sqi` runs the official OpenJD conformance test suite on every CI build and publishes the current pass rate and known gaps at [`docs/openjd-conformance.md`](docs/openjd-conformance.md). One caveat is by design rather than a gap: a template that opts into an OpenJD extension `sqi` does not implement is rejected outright, rather than being accepted and misinterpreted. The official `EXPR` (expression language) extension **is** implemented and supported — see [`docs/openjd-extensions/expr.md`](docs/openjd-extensions/expr.md). **General purpose.** Rendering is the primary use case and the domain `sqi` is designed around, but the job model is general. Any workload expressible as a command with defined inputs, outputs, and environment is a valid `sqi` job — simulation, transcoding, machine learning pipelines, data processing, software development, or anything else a studio runs at scale. diff --git a/ROADMAP.md b/ROADMAP.md index bfc31575..db4e71b2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -88,7 +88,7 @@ Configuration cascades: farm defaults → queue overrides, with retry policy (ma reservation; omitting it reserves the whole machine (one task per worker). The server tracks committed cores in the database; the ledger rebuilds instantly on restart. - - *Deferred:* worker drain/headroom signal, head-of-line reservation for + - *Deferred:* worker drain/headroom signal, head-of-line reservation for large tasks, `amount.worker.vcpu.max`, memory/GPU dimensions. --- @@ -98,7 +98,7 @@ Configuration cascades: farm defaults → queue overrides, with retry policy (ma `sqi` adopts the [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications) (OpenJD) format as its native job execution format. **Benefits:** -- Studios authoring jobs for other OpenJD-compatible systems can submit to `sqi` unchanged, provided the template does not opt into an extension `sqi` has not implemented (e.g. `EXPR`) — those are rejected by design rather than accepted and misinterpreted — `sqi` accepts every valid base-spec template in the official conformance suite, though it is still more permissive than the spec about rejecting *invalid* ones (tracked in `test/conformance/baseline.txt`, measured in [`docs/openjd-conformance.md`](docs/openjd-conformance.md)) +- Studios authoring jobs for other OpenJD-compatible systems can submit to `sqi` unchanged, provided the template does not opt into an extension `sqi` has not implemented — those are rejected by design rather than accepted and misinterpreted (the official `EXPR` expression-language extension **is** implemented and supported) — `sqi` accepts every valid base-spec template in the official conformance suite, though it is still more permissive than the spec about rejecting *invalid* ones (tracked in `test/conformance/baseline.txt`, measured in [`docs/openjd-conformance.md`](docs/openjd-conformance.md)) - Standardized path mapping, parameter spaces, and execution semantics - Clear separation between job description and job authoring (the product system) @@ -245,7 +245,7 @@ NATS can run embedded within `sqi-server` (simple mode) or as a separate cluster - S3-compatible storage support (thin layer: derived type, root validation, path staging via operator sync tool) - DCC submitter framework — in-application submitters for Maya, Houdini, Nuke, and Blender (the `sqi-submitter` Python package), built on the Python client - Compute location registry and step-level affinity (native OpenJD `attr.worker.computelocation`) -- Chunk bounds (`SQI_CHUNK_BOUNDS` vendor extension) — expose each task chunk's frame start/end to the command line, used by the Maya and Blender reference presets +- Chunk bounds (`SQI_CHUNK_BOUNDS` vendor extension) — expose each task chunk's frame start/end to the command line, used by the Maya, Blender and Mistika reference presets - Auto-retry and failure limits — per-task retry policy (max attempts, retry delay) and a job-level failure ceiling that auto-parks a job, resolved over four tiers (server → farm → queue → job) with per-task attempt history - Cross-job dependencies — a submission may declare `depends_on` upstream jobs (same farm, across queues); dependents are held `blocked` until every upstream completes, then released (or canceled if an upstream fails) - Testing job presets — ready-to-run `test-render`/`test-steps` presets (bash and PowerShell) published to the preset library for smoke-testing a farm @@ -271,6 +271,23 @@ before Phase 3. See [docs/auth.md](docs/auth.md) for the model and setup. [docs/worker-configuration.md](docs/worker-configuration.md) for the model, setup, and known gaps. +### Post-Phase-3, pre-v0.3: OpenJD `EXPR` and expanded reference presets — complete, unreleased + +- **OpenJD `EXPR` extension** — the official expression-language extension is + fully implemented and `StatusSupported`: expression core, type system, + collections, comprehensions, the ~100-function standard library, path + mapping, template integration (scopes, `let` bindings, bounded evaluation + with operator-configurable limits), RFC 0007 extended parameter types, and + the web `*_LIST` widgets. EXPR templates are accepted, submitted, dispatched + and executed, with expressions resolved on the worker at phase 3. See + [`docs/openjd-extensions/expr.md`](docs/openjd-extensions/expr.md). +- **ffmpeg reference presets** — transcode, sequence-encode, and + segment-transcode (bash/PowerShell/EXPR) added to `presets/sqi/`; the + segment-transcode EXPR variant is the first shipped preset to declare + `extensions: [EXPR]`. +- **Mistika reference presets** — Boutique, VR, and Workflows render presets + added to `presets/sqi/`, each using the `SQI_CHUNK_BOUNDS` extension. + ### Phase 4: Production Hardening (v0.4 — beta) - PostgreSQL state store option diff --git a/SECURITY.md b/SECURITY.md index 198f78af..cedb56fb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,7 +19,7 @@ Once stable releases begin, this table will be updated to reflect which release Use GitHub's built-in private reporting to submit a vulnerability report confidentially: -1. Go to the [sqi Security Advisories page](../../security/advisories/new) +1. Go to the [sqi Security Advisories page](https://github.com/uberware/sqi/security/advisories/new) 2. Click **Report a vulnerability** 3. Fill in the details — affected component, reproduction steps, potential impact, and any suggested mitigations you have in mind @@ -65,8 +65,9 @@ The following are in scope for security reports: - `sqi-server` — scheduler, REST API, WebSocket, embedded NATS, SQLite state management - `sqi-worker` — task executor and worker agent -- The Python client library (`sqi-sdk`) -- The authentication and authorization implementation (once shipped — Phase 3+) +- The Python client library (`sqi-sdk`) and the DCC submitter package (`sqi-submitter`) +- The authentication and authorization implementation (`internal/auth/`) — the auth gate, local accounts, sessions, API keys, RBAC, LDAP/AD, and OAuth2/OIDC SSO +- Run-as-user task isolation (`internal/worker/isolation/`) — privilege separation between task processes and the worker service account - Dependency vulnerabilities in the Go module graph or npm packages The following are **out of scope**: diff --git a/api/README.md b/api/README.md index 6e2a6ba1..aa74ab56 100644 --- a/api/README.md +++ b/api/README.md @@ -1,5 +1,13 @@ # `api/` -API contract artifacts that are not Go source: the OpenAPI 3.1 specification, JSON Schemas for OpenJD payloads and config files, and any future Protobuf / gRPC definitions for the worker wire protocol. +Reserved for API contract artifacts that are not Go source — JSON Schemas for +OpenJD payloads and config files, and any future Protobuf / gRPC definitions for +the worker wire protocol. -Generated client/server code lives next to the consumers (`internal/api/...`, the Python client, etc.), not here. This directory holds the source-of-truth specs. +**The directory is currently empty.** The authoritative OpenAPI 3.1 +specification lives at [`internal/api/openapi.yaml`](../internal/api/openapi.yaml), +next to the handlers it describes; it is embedded into `sqi-server` and served +at `GET /api/v1/openapi.yaml`. + +Generated client/server code lives next to its consumers (`internal/api/...`, +`clients/python/`, …), not here. diff --git a/clients/python/README.md b/clients/python/README.md index e6610e96..a37ccf63 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -28,8 +28,8 @@ pip install 'sqi-sdk[yaml]' # + PyYAML (for your own YAML handling; not neede pip install 'sqi-sdk[ws]' # + websockets for live event streaming ``` -Until the package is published to PyPI, install the wheel attached to a -[GitHub release](https://github.com/uberware/sqi/releases): +For offline installs or a specific pre-release build, the wheel is also attached +to each [GitHub release](https://github.com/uberware/sqi/releases): ```sh pip install https://github.com/uberware/sqi/releases/download/vX.Y.Z/sqi_sdk-X.Y.Z-py3-none-any.whl @@ -117,7 +117,8 @@ user — without it the server responds 403. ## Products Products are named, versioned wrappers around OpenJD templates that live in the -server's catalog. The SDK exposes seven methods for working with them: +server's catalog. The SDK exposes eight methods for working with them +(`iter_products()` is the iterator companion to `list_products()`): ```python with SqiClient("http://localhost:8080") as sqi: diff --git a/clients/submitter/README.md b/clients/submitter/README.md index ac47fb5b..8fe927f5 100644 --- a/clients/submitter/README.md +++ b/clients/submitter/README.md @@ -15,13 +15,23 @@ pip install sqi-submitter # core (requires sqi-sdk) pip install 'sqi-submitter[qt]' # + PySide6 for standalone UI ``` -For development: +For development (run from `clients/submitter/`): ```sh -pip install -e '.[dev]' # with test & lint tools -pip install -e ../python # resolve sqi-sdk from local checkout +pip install -e ../python -e '.[dev]' # local sqi-sdk + submitter with test & lint tools ``` +The same checks CI runs (split there across three jobs), as one local chain — +every stage must pass before a PR: + +```sh +ruff format --check . && ruff check . && mypy src && mypy --python-version=3.13 tests && pytest -q +``` + +`mypy --python-version=3.13 tests` is not optional: `pyproject.toml` scopes +mypy to `src`, so running `mypy src` alone misses type errors in test files +that will still fail the PR. + ## Authentication When the target `sqi-server` has `auth.enabled=true` (see @@ -36,7 +46,18 @@ forwarded to `SqiClient` as the Bearer token. Issue a key for yourself via ## Quick start -Use the `fake_host_module` fixture in tests to inject mock DCC modules: +Open the standalone dialog (needs the `qt` extra, or a DCC-bundled PySide): + +```sh +sqi-submit --server http://localhost:8080 +``` + +Inside a DCC, wire the host's launch glue per +[`docs/dcc-submitters.md`](../../docs/dcc-submitters.md#installation-per-host). + +## Testing + +Use the `fake_host_module` fixture to inject mock DCC modules: ```python def test_with_maya(fake_host_module): diff --git a/cmd/sqi-server/README.md b/cmd/sqi-server/README.md index 618e1901..6a2b7d78 100644 --- a/cmd/sqi-server/README.md +++ b/cmd/sqi-server/README.md @@ -16,14 +16,14 @@ interact with it over HTTP and WebSocket. | **WebSocket gateway** | `/api/v1/ws` endpoint for real-time push of job, task, worker, and log events to subscribed clients. | | **Embedded NATS** | In-process JetStream broker used for work assignment, status reporting, log streaming, and worker heartbeats. | | **SQLite store** | Single-file database holding all durable state — jobs, tasks, workers, farms, queues, usage pools, and audit log. | -| **Web UI host** | Serves the embedded SPA at `/ui/` and the OpenAPI spec at `/api/v1/openapi.yaml`. | +| **Web UI host** | Serves the embedded SPA at the root (`/`, with SPA fallback for extensionless paths; the legacy `/ui/*` prefix still resolves) and the OpenAPI spec at `/api/v1/openapi.yaml`. | | **mDNS responder** | Advertises `_sqi._tcp` on the local network so workers and the CLI can discover the server without manual address configuration. | --- ## Build -**Prerequisites:** Go 1.23 or newer (version is pinned in `go.mod`). +**Prerequisites:** Go 1.26 or newer (the `go` directive in `go.mod` pins 1.26.3). ```sh # Build just the server binary into ./bin/ @@ -97,7 +97,11 @@ curl -sf http://localhost:8080/readyz # expects: {"status":"ok","checks":{"na **3. Submit a minimal OpenJD job** ```sh -JOB_ID=$(curl -sf -X POST http://localhost:8080/api/v1/jobs \ +FARM=$(curl -sf http://localhost:8080/api/v1/farms | jq -r '.[0].id') +QUEUE=$(curl -sf "http://localhost:8080/api/v1/queues?farm_id=$FARM" | jq -r '.items[0].id') + +JOB_ID=$(curl -sf -X POST \ + "http://localhost:8080/api/v1/jobs?farm_id=$FARM&queue_id=$QUEUE&owner=smoke" \ -H 'Content-Type: application/yaml' \ --data-binary ' specificationVersion: "jobtemplate-2023-09" @@ -123,7 +127,7 @@ curl -sf http://localhost:8080/api/v1/jobs/$JOB_ID | jq '{id, name, status}' **5. List tasks** ```sh -curl -sf http://localhost:8080/api/v1/jobs/$JOB_ID/tasks | jq '[.[] | {id, status}]' +curl -sf http://localhost:8080/api/v1/jobs/$JOB_ID/tasks | jq '[.items[] | {id, status}]' ``` **6. Check Prometheus metrics** @@ -180,11 +184,11 @@ internal/ ├── health/ /healthz and /readyz handlers. ├── log/ slog-based structured logger and request-scoped middleware. ├── metrics/ Prometheus metric definitions and the /metrics handler. -├── middleware/ HTTP middleware (recovery, CORS, request ID, gzip, logger). -├── openjd/ OpenJD v2025-09 parser, validator, parameter-space expansion. -├── scheduler/ Assignment loop, worker registry, task state machine. +├── middleware/ HTTP middleware (recovery, CORS, request ID, gzip, logger). +├── openjd/ OpenJD (jobtemplate-2023-09) parser, validator, parameter-space expansion. +├── scheduler/ Assignment loop, worker registry, heartbeat sweep, retry/failure policy. ├── server/ HTTP server wiring — router, middleware stack, boot sequence. -├── store/ Store interface, SQLite implementation, migration runner. +├── store/ Store interface, SQLite implementation, migration runner, task state machine. ├── ui/ Embedded web/dist assets, SPA fallback handler. ├── version/ Build metadata (version, commit, build date, Go version). ├── worker/ Worker wire protocol handlers and log ingestion. diff --git a/cmd/sqi-worker/README.md b/cmd/sqi-worker/README.md index faaa0412..e58d76da 100644 --- a/cmd/sqi-worker/README.md +++ b/cmd/sqi-worker/README.md @@ -1,8 +1,9 @@ # sqi-worker `sqi-worker` is the distributed task-execution agent for the sqi render farm. It -connects to a running `sqi-server`, registers its hardware capabilities, and pulls -task assignments from NATS JetStream, executing them as bare-metal OS processes. +connects to a running `sqi-server`, registers its hardware capabilities, and +requests task assignments over core-NATS work leases (`work.lease.`), +executing them as bare-metal OS processes. --- @@ -13,8 +14,10 @@ task assignments from NATS JetStream, executing them as bare-metal OS processes. - **Registers** its capabilities at startup — OS, CPU count, RAM, GPU (where detectable), plus any manual tags from configuration such as `maya-2025` or `arnold-7`. -- **Pulls** task assignments over NATS JetStream and executes them concurrently; - the server gates concurrency via CPU-core accounting. +- **Leases** task assignments over core NATS (a long-polling request/reply on + `work.lease.`) and executes them concurrently; the server gates + concurrency via CPU-core accounting. Task status, logs, heartbeats and + registration travel the other way over JetStream. - **Streams** task stdout and stderr back to `sqi-server` in real time so the web UI log viewer feels live. - **Interprets** OpenJD progress directives (`openjd_progress`, @@ -142,8 +145,14 @@ curl -s http://localhost:8080/api/v1/workers | jq '.[].name' | `sqi-worker start --dry-run` | Validate config and capabilities without connecting | | `sqi-worker start --nats-insecure-skip-verify` | Skip TLS cert verification (dev only) | | `sqi-worker config print` | Print the effective merged configuration | +| `sqi-worker capabilities` | Print every tag this worker would advertise, with its source | +| `sqi-worker isolation set-credential ` | Store a run-as-user account's password (Windows; reads the secret from stdin) | | `sqi-worker version` | Print version, commit, build date, and Go version | +Three flags are available on **every** subcommand: `--config`/`-c` (config file +path), `--log-level`, and `--log-format`. `--dry-run` and +`--nats-insecure-skip-verify` belong to `start` only. + --- ## See also diff --git a/cmd/sqi-worker/main.go b/cmd/sqi-worker/main.go index f42764bd..b1400e26 100644 --- a/cmd/sqi-worker/main.go +++ b/cmd/sqi-worker/main.go @@ -3,8 +3,9 @@ // Command sqi-worker is the sqi distributed task management worker agent. // // It discovers and connects to a running sqi-server, registers itself with -// its capability tags and compute location, pulls task assignments over NATS -// JetStream, and executes bare-metal OS processes inside OpenJD sessions. +// its capability tags and compute location, leases task assignments over core +// NATS (work.lease.), and executes bare-metal OS processes inside +// OpenJD sessions. // // Run "sqi-worker --help" for usage. package main diff --git a/cmd/sqi-worker/root.go b/cmd/sqi-worker/root.go index efd4e12e..46e5985f 100644 --- a/cmd/sqi-worker/root.go +++ b/cmd/sqi-worker/root.go @@ -16,8 +16,9 @@ var rootCmd = &cobra.Command{ Long: `sqi-worker is the worker agent for the sqi distributed task management platform. It discovers and connects to a running sqi-server, registers itself with its -capability tags and compute location, pulls task assignments over NATS -JetStream, and executes bare-metal OS processes inside OpenJD sessions. +capability tags and compute location, leases task assignments over core NATS +(work.lease.), and executes bare-metal OS processes inside OpenJD +sessions. Use "sqi-worker start" to start the worker agent. Use "sqi-worker --help" for a list of available subcommands.`, diff --git a/config/sqi-server.example.yaml b/config/sqi-server.example.yaml index 5af5240e..c1a54b6c 100644 --- a/config/sqi-server.example.yaml +++ b/config/sqi-server.example.yaml @@ -61,12 +61,13 @@ store: # Type: string Env: SQI_STORE_SQLITE_PATH sqlite_path: "sqi.db" - # How often the background WAL checkpointer runs PRAGMA wal_checkpoint(TRUNCATE) - # to fold committed WAL frames back into the main database file and truncate - # the WAL. Without periodic checkpointing the WAL grows unboundedly under - # write load. A final checkpoint always runs on clean shutdown regardless of - # this interval. Set to a large value (e.g. "24h") to disable periodic - # checkpointing while keeping the shutdown checkpoint. + # How often the background WAL checkpointer runs PRAGMA wal_checkpoint(PASSIVE) + # to fold committed WAL frames back into the main database file. Without + # periodic checkpointing the WAL grows unboundedly under write load. The + # periodic checkpoint is PASSIVE so it never waits on a reader; a final + # TRUNCATE checkpoint always runs on clean shutdown regardless of this + # interval, leaving the WAL at zero bytes. Set to a large value (e.g. "24h") + # to disable periodic checkpointing while keeping the shutdown checkpoint. # Must be > 0. # Type: duration Env: SQI_STORE_CHECKPOINT_INTERVAL checkpoint_interval: "5m" diff --git a/config/sqi-worker.example.yaml b/config/sqi-worker.example.yaml index 85d9bcef..d3ba4db8 100644 --- a/config/sqi-worker.example.yaml +++ b/config/sqi-worker.example.yaml @@ -158,14 +158,14 @@ worker: # Type: bool Env: SQI_WORKER_KEEP_FAILED_SESSIONS keep_failed_sessions: false - # Restrict this worker to serving specific queue IDs. When empty (the - # default) the worker accepts assignments from all queues via a wildcard - # JetStream consumer. Set this on heterogeneous farms where some workers - # specialise in a subset of queues, e.g. separate GPU and CPU queues. - # - # IMPORTANT: Do not mix wildcard (empty queue_ids) and per-queue workers on - # the same SQI_WORK JetStream stream — both consumer types would race on - # overlapping messages. Choose one strategy for your farm. + # Restrict this worker to serving specific queue IDs. The worker keeps one + # outstanding lease request per listed queue (work.lease.). When + # empty (the default) it issues a single lease request on the reserved + # core-NATS subject work.lease._any — the server selects tasks farm-wide for + # that token and gates by worker eligibility, so a queue-unaffiliated worker + # is matched to any queue's ready work. Set this on heterogeneous farms where + # some workers specialise in a subset of queues, e.g. separate GPU and CPU + # queues. # # Type: []string Env: SQI_WORKER_QUEUE_IDS (comma-separated) queue_ids: [] @@ -174,20 +174,20 @@ worker: # - gpu-renders # - cpu-preview - # Wait duration between pull attempts when the work queue is empty. Prevents - # tight polling loops on idle queues. Resets to zero immediately when a task - # is received. Increase if many idle workers are generating NATS traffic; - # decrease if you need lower task start latency on bursty queues. - # Must be > 0. + # DEPRECATED. Accepted by the config loader for backwards compatibility, but + # it has NO EFFECT in the current lease-based worker — changing it tunes + # nothing. Idle backoff is no longer needed: the worker's lease request + # long-polls on the server (~30 s hold) and re-issues immediately on return, + # so there is no tight polling loop to throttle. Only a negative value is + # still rejected at load. # Type: duration Env: SQI_WORKER_PULL_IDLE_BACKOFF pull_idle_backoff: "2s" - # Redelivery delay applied to an assignment when pre-execution validation - # fails (e.g., compute location mismatch, protocol version incompatibility). - # The delay gives other workers a window to claim the assignment before NATS - # redelivers it to this worker. Set higher if transient failures are frequent - # and you want to reduce retry churn. - # Must be > 0. + # DEPRECATED. Accepted by the config loader for backwards compatibility, but + # it has NO EFFECT in the current lease-based worker — changing it tunes + # nothing. Pre-execution NACKs no longer apply: the server validates + # eligibility before leasing, so the worker runs what it is given. Only a + # negative value is still rejected at load. # Type: duration Env: SQI_WORKER_PULL_NACK_DELAY pull_nack_delay: "5s" @@ -390,15 +390,20 @@ diagnostics: # the sum of every environment it enters, so sizing host RAM against # assignment_retained_bytes over-provisions, and sizing it against an # observed RSS under-bounds it. -# * TIGHTENING IS NOT FREE HERE THE WAY IT IS ON THE SERVER. Four of these -# five have a server-side counterpart that meters the same thing one phase +# * TIGHTENING IS NOT FREE HERE THE WAY IT IS ON THE SERVER. Every one of +# these five has a server-side limit metering the same values one phase # earlier, and this worker must not be TIGHTER than the server it reports # to: # operation_limit >= server openjd.expr_operation_limit # memory_limit >= server openjd.expr_memory_limit # assignment_positions >= server openjd.expr_template_positions # assignment_retained_bytes >= server openjd.expr_template_retained_bytes -# This worker advertises all four at registration and the server REFUSES TO +# let_retained_bytes >= server openjd.expr_template_retained_bytes +# The last row pairs a per-table limit with a template-wide one, because the +# server meters no per-table scope; a whole template's retained bytes is a +# valid upper bound on any one of its tables, so the comparison is +# conservative. +# This worker advertises all five at registration and the server REFUSES TO # DISPATCH EXPR jobs to it while any of them is short — it does not accept # the job and then fail it here, once per task. The cost of tightening past # the server is therefore that this host stops being offered EXPR work @@ -407,7 +412,6 @@ diagnostics: # exists for — the latter only while the server's unschedulable sweep is on # (its scheduler.unschedulable_grace > 0, the default); with the sweep off, # such a task waits with nothing written on it. -# let_retained_bytes has no server counterpart and is not advertised. # # An out-of-range value is a startup failure, not a clamp. expr: diff --git a/deploy/README.md b/deploy/README.md index ce8d273a..69dd17be 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -9,7 +9,7 @@ Deployment artifacts for `sqi`: - `deploy/config/` — pointer to the canonical example configs under `config/`. Kubernetes manifests / a Helm chart for production-mode deployments are planned -for a later phase and are not part of v0.1.0. +for a later phase and are not shipped yet. Goreleaser configuration lives at the repo root as `.goreleaser.yaml` to match tool convention. diff --git a/docs/README.md b/docs/README.md index 7bc35330..5c51f4a6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,7 @@ Long-form documentation for `sqi-server` operators and contributors. - `quickstart.md` — end-to-end first run (binary or Docker Compose): farm, queue, first job - `index.md` — overview / landing page (mirrors the repo-root `README.md` for the docs site) +- `examples/` — runnable sample job payloads (e.g. `hello.json`, used by the release runbook) **Architecture & API** @@ -14,6 +15,7 @@ Long-form documentation for `sqi-server` operators and contributors. - `python-client.md` — `sqi-sdk` Python library reference (construction, every method, examples) - `openjd-submission.md` — minimal, parameter-space, and multi-step OpenJD examples - `openjd-extensions.md` — the OpenJD extension registry and how to add one (subdocs under `openjd-extensions/`) +- `openjd-conformance.md` — the official OpenJD conformance suite results, current pass rate, and known gaps **Products, presets & submitters** (Phase 2) @@ -53,4 +55,4 @@ Long-form documentation for `sqi-server` operators and contributors. - `roadmap.md` — technical-architecture and roadmap reference (mirrors the repo-root `ROADMAP.md`) - `spdx-header.md` — the SPDX license-header convention for source files -The canonical product/vision docs (`README.md`, `ROADMAP.md`, `CONTRIBUTING.md`) live at the repo root; `docs/index.md`, `docs/roadmap.md`, and `docs/contributing.md` are docs-site copies with site-relative links. `ROADMAP.md` is the technical-architecture and roadmap reference these docs point to for design rationale. +The canonical product/vision docs (`README.md`, `ROADMAP.md`, `CONTRIBUTING.md`) live at the repo root; `docs/index.md`, `docs/roadmap.md`, and `docs/contributing.md` are docs-site copies whose **only** intended difference is site-relative links. Any edit to a root copy must be mirrored into its `docs/` twin in the same commit — the two roadmap copies have already drifted apart in both directions. diff --git a/docs/api.md b/docs/api.md index db055acf..dd0eb4e2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -15,7 +15,7 @@ You can browse it with any OpenAPI viewer (e.g. Swagger UI, Redoc, or ## Base URL -All REST endpoints share the prefix `/api/v1`. A locally-running server +All REST endpoints share the prefix `/api/v1`. A locally running server listens on `http://localhost:8080` by default. ``` @@ -47,7 +47,7 @@ problem-details format with `Content-Type: application/problem+json`: ``` The `instance` field contains the request ID, which also appears in the -`X-Request-Id` response header — useful when correlating with server logs. +`X-Request-ID` response header — useful when correlating with server logs. ### Rate limiting @@ -66,8 +66,9 @@ Clients should back off for at least the advertised duration before retrying. ### Pagination -List endpoints accept `limit` (default 50, max 1000) and `offset` -(zero-based) query parameters and return a wrapper object: +Jobs, tasks, queues, and workers are paginated: those list endpoints accept +`limit` (default 50, max 1000) and `offset` (zero-based) query parameters and +return a wrapper object: ```json { @@ -78,6 +79,10 @@ List endpoints accept `limit` (default 50, max 1000) and `offset` } ``` +Other list endpoints are **not** paginated — including farms, storage +locations, compute locations, usage pools, products, presets, users, and API +keys. Those ignore `limit`/`offset` and return a bare JSON array. + ### Versioning The URL prefix is the API contract version, and every response under @@ -103,6 +108,54 @@ Clients should warn when they see a `Deprecation` header or an --- +## Authentication + +Authentication is **off by default** (`auth.enabled=false`), in which case every +request is an anonymous superuser and no credential is needed — every example +below works as written. See [`docs/auth.md`](auth.md) for the full model. + +When `auth.enabled=true`, send an API key as a bearer token: + +```sh +curl -s -H "Authorization: Bearer $SQI_API_KEY" "$BASE/jobs" | jq . +``` + +Browser clients use the session cookie minted by `POST /api/v1/auth/login` +instead. Three endpoints are always public because gating them would be +circular: `GET /api/v1/openapi.yaml`, `POST /api/v1/auth/login`, and +`GET /api/v1/auth/providers` (plus `GET /api/v1/auth/oidc/login` and +`GET /api/v1/auth/oidc/callback` when SSO is configured). + +Every other `/api/v1` REST endpoint requires a permission (`/healthz`, +`/readyz` and `/metrics` sit outside the API prefix and are never gated; the +`/ws` upgrade authenticates the same way but gates per-subject — see +[WebSocket subscriptions](#websocket-subscriptions)): + +| Endpoint group | Permission | +|---|---| +| `GET /jobs`, `GET /jobs/{id}`, `GET /jobs/{id}/tasks`, `GET /tasks/{id}`, `GET /tasks/{id}/logs`, `GET /tasks/{id}/attempts` | `jobs.read` | +| `POST /jobs`, `POST /products/{name}/jobs`, `PATCH/DELETE /jobs/{id}`, `POST /jobs/{id}/cancel`, `POST /jobs/{id}/retry`, `POST /tasks/{id}/retry`, `POST /tasks/{id}/cancel` | `jobs.write` | +| `GET /workers`, `GET /workers/{id}` | `workers.read` | +| `POST /workers/{id}/disable`, `POST /workers/{id}/enable`, `DELETE /workers/{id}` | `workers.manage` | +| `GET` on farms, queues, storage-locations, compute-locations, usage-pools | `infra.read` | +| `POST`/`PUT`/`DELETE` on farms, queues, storage-locations, compute-locations, usage-pools | `infra.manage` | +| `GET /products`, `GET /products/{name}`, `GET /products/{name}/parameters`, `GET /presets`, `GET /presets/{name}` | `products.read` | +| `POST /products`, `PUT /products/{name}`, `DELETE /products/{name}`, `POST /presets/{name}/install` | `products.manage` | +| `GET /diagnostics/logs`, WebSocket subject `diagnostics` | `diagnostics.read` | +| `GET /users`, `GET /users/{id}` | `users.read` | +| `POST /users`, `PATCH /users/{id}`, `PUT /users/{id}/password`, `DELETE /users/{id}` | `users.manage` | +| `POST/GET /api-keys`, `DELETE /api-keys/{id}` | `apikeys.self` | +| `GET /users/{id}/api-keys`, `DELETE /users/{id}/api-keys/{keyId}` | `apikeys.admin` | +| `POST /auth/logout`, `GET/PATCH /auth/me`, `PUT /auth/password`, `GET /version` | any authenticated principal | + +Object routes (`/jobs/{id}…`, `/tasks/{id}…`) are additionally owner-scoped: a +principal without `jobs.read.all` sees and acts on only its own jobs — the one +permission governs both read and write scoping. A missing or rejected +credential returns `401`; a valid credential without the permission returns +`403`, as does an owner-scoped principal reaching another user's job. + +--- + ## Worked examples The examples below use `curl`. Replace `$BASE` with `http://localhost:8080/api/v1` @@ -130,8 +183,8 @@ Optional query parameters: `owner`, `submitter`, `priority` (default 50, higher `retry_delay_seconds` (≥ 0), and `failure_limit` (≥ 0; the job-level failure ceiling that auto-parks the job, 0 disables an inherited limit) — each omitted means inherit the queue → farm → server default, and an out-of-range value is -rejected with 400. Also `depends_on` (repeatable) — IDs of upstream jobs, in the same -farm, this job must wait for; if any is not yet `completed` the job is created +rejected with 400. Finally, `depends_on` (repeatable) — IDs of upstream jobs, in +the same farm, this job must wait for; if any is not yet `completed` the job is created `blocked` instead of `pending` and its tasks are held until every dependency completes (see [`docs/architecture.md`](architecture.md#job-lifecycle-data-flow)). The same `depends_on` field is accepted in the JSON body when submitting from @@ -162,7 +215,8 @@ Successful response — `201 Created`: "status": "pending", "template_format": "yaml", "created_at": "2026-01-15T10:00:00Z", - "updated_at": "2026-01-15T10:00:00Z" + "updated_at": "2026-01-15T10:00:00Z", + "failed_attempts": 0 } ``` @@ -178,6 +232,22 @@ Validation failure — `422 Unprocessable Entity`: } ``` +Server gave up evaluating the template's expressions — `503 Service Unavailable`: + +```json +{ + "type": "about:blank", + "title": "Service Unavailable", + "status": 503, + "detail": "template validation exceeded its time budget on this server; retry, or ask the operator about openjd.expr_submission_deadline", + "instance": "a1b2c3d4e5f60708" +} +``` + +This is **not** a verdict on the template: the deterministic expression budgets +report an invalid template as `422`, while this outcome depends on how busy the +server was, so the same body may well be accepted on a retry. + --- ### Submit a job from a product @@ -196,9 +266,8 @@ The `name` field overrides the template's job name; when omitted the template's own name is used. `farm_id` and `queue_id` are required. `parameters` is a flat string→string map matching the template's `parameterDefinitions`. `depends_on` is an optional array of upstream job IDs, same semantics as the raw-submit -query parameter above. The -`/parameters` endpoint returns each parameter's type, default, allowed values, -and `user_interface` hints. +query parameter above. The `/parameters` endpoint returns each parameter's +type, default, allowed values, and `user_interface` hints. --- @@ -215,6 +284,7 @@ Filter parameters (all optional): | `farm_id` | UUID | Filter by farm | | `owner` | string | Filter by owner | | `project` | string | Filter by project | +| `search` | string | Case-insensitive substring over name, id, owner, project. Whitespace-separated words are matched as independent terms, ANDed and order-independent (e.g. `night alice` matches a job named "Nightly" owned by "alice") | | `sort_by` | `created_at`, `priority`, `status`, `updated_at`, `name` | Sort field (default: `created_at`) | | `sort_dir` | `asc`, `desc` | Sort direction (default: `asc`) | | `limit` | 1–1000 | Page size (default: 50) | @@ -240,6 +310,10 @@ carries `effective_retry` — the resolved retry policy for this job server → farm → queue → job cascade — plus `failed_attempts` and, when the job has been auto-parked, a `park_reason`. +`task_counts.unschedulable` counts `ready` tasks that currently carry a +non-empty unschedulable reason — a subset of `ready`, not an additional status, +so it is not included in `total` a second time. + ```sh JOB_ID=018f1a2b-3c4d-7e5f-a6b7-c8d9e0f12345 @@ -258,7 +332,8 @@ curl -s "$BASE/jobs/$JOB_ID" | jq '{name, status, task_counts}' "running": 5, "succeeded": 80, "failed": 0, - "canceled": 0 + "canceled": 0, + "unschedulable": 0 } } ``` @@ -279,11 +354,57 @@ curl -s "$BASE/jobs/$JOB_ID/tasks?status=failed" | jq .items[] --- +### Get a task + +`GET /api/v1/tasks/{id}` + +Returns a single task. Returns `404` if no task with that ID exists. + +```sh +TASK_ID= + +curl -s "$BASE/tasks/$TASK_ID" | jq . +``` + +```json +{ + "id": "...", + "job_id": "...", + "step_id": "...", + "name": "Render-1", + "parameters": { "Frame": "1" }, + "status": "failed", + "assigned_worker_id": "worker-abc", + "assigned_at": "2026-01-15T10:05:40.000Z", + "created_at": "2026-01-15T10:00:00Z", + "updated_at": "2026-01-15T10:05:45.000Z", + "failure_reason": "openjd_fail: step action returned non-zero", + "failed_attempts": 1 +} +``` + +The worker that took the task is `assigned_worker_id`. The bare name +`worker_id` means the same thing elsewhere — on the WebSocket +`jobs/{job-id}/tasks` push payload and on each entry of +`GET /tasks/{id}/attempts` — but never on this response. +`unschedulable_reason` is non-empty only while a `ready` task cannot be +satisfied by any online worker. `failure_reason` is the task-level reason for a +terminal non-success and is **cleared on retry** — the per-attempt `message` +from `GET /tasks/{id}/attempts` is the durable record. `retry_after`, when set +and in the future, holds a `ready` task as a retry backoff. `parameters`, +`assigned_worker_id`, `assigned_at`, `unschedulable_reason`, `failure_reason`, +and `retry_after` are omitted when empty. + +--- + ### Cancel a job `POST /api/v1/jobs/{id}/cancel` Cancels the job and propagates cancel signals to all assigned workers. +Idempotent: an already-canceled job returns `204`. Returns `404` if the job +does not exist, and `409` if the job has already reached `completed` or +`failed`. ```sh curl -s -X POST "$BASE/jobs/$JOB_ID/cancel" @@ -348,13 +469,20 @@ curl -s -X PATCH "$BASE/jobs/$JOB_ID" \ curl -s -X PATCH "$BASE/jobs/$JOB_ID" \ -H "Content-Type: application/json" \ -d '{"max_attempts": 5, "retry_delay_seconds": 60, "failure_limit": 10}' + +# Move the job to a different queue +curl -s -X PATCH "$BASE/jobs/$JOB_ID" \ + -H "Content-Type: application/json" \ + -d '{"queue_id": ""}' ``` -The same endpoint sets the per-job retry-policy overrides (`max_attempts`, -`retry_delay_seconds`, `failure_limit`); omitting a field (or sending `null`) -leaves it unchanged — a job-level override cannot be cleared back to "inherit" -this way. Resuming an auto-parked job (`action: resume`) also clears -`park_reason` and resets `failed_attempts` to zero, re-arming its failure limit. +The same endpoint moves a job between queues (`queue_id`; a queue that does not +exist is rejected with `400`) and sets the per-job retry-policy overrides +(`max_attempts`, `retry_delay_seconds`, `failure_limit`); omitting a field (or +sending `null`) leaves it unchanged — a job-level override cannot be cleared +back to "inherit" this way. Resuming an auto-parked job (`action: resume`) also +clears `park_reason` and resets `failed_attempts` to zero, re-arming its failure +limit. --- @@ -367,8 +495,9 @@ cursor using `after_nats_seq` from each response. | Parameter | Default | Description | |---|---|---| -| `limit` | 100 | Chunks per page | +| `limit` | 100 | Chunks per page (1–1000) | | `after_nats_seq` | 0 | Return only chunks with NATS sequence > this value | +| `tail` | `false` | When `true`, stream chunks live as newline-delimited JSON (`Content-Type: application/x-ndjson`) instead of returning a page. Streaming stops when the task reaches a terminal state and all buffered chunks have been delivered, or when the client disconnects. | ```sh TASK_ID= @@ -418,7 +547,7 @@ Quick example using [`websocat`](https://github.com/vi/websocat): TASK_ID= websocat ws://localhost:8080/api/v1/ws <`, and `attrs` is omitted when the record carries no +structured attributes. + +On both the `jobs` and `workers` subjects a `status` of `removed` is a +synthetic value (not a persisted job/worker status) meaning the row was +hard-deleted — drop it from your view rather than updating it in place. + Example `push` frame: ```json diff --git a/docs/architecture.md b/docs/architecture.md index 24b80c0d..c3829a45 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,8 +30,9 @@ through scheduling, worker execution, and final state. │ │ embedded NATS (JetStream + core NATS) │ │ │ │ │ │ │ │ work.lease. task.status. │ │ -│ │ task.logs. worker.heartbeat │ │ -│ │ worker.register │ │ +│ │ task.logs. task.cancel. │ │ +│ │ worker.register worker.heartbeat │ │ +│ │ worker.deregister worker.diag. │ │ │ └────────┬────────────────────────────────────────────────┬─┘ │ │ │ │ │ │ ┌───────────▼──────────┐ ┌────────────────▼──┐ │ @@ -60,6 +61,9 @@ through scheduling, worker execution, and final state. | Scheduler | `internal/scheduler` | Assignment loop, worker registry, heartbeat sweep, usage pool gating | | NATS bus | `internal/bus` | Typed JetStream client wrapper; stream, subject, and consumer definitions | | Store | `internal/store` | `Store` interface + SQLite implementation; migrations | +| Diagnostics | `internal/diag` | Bounded, per-component in-memory ring buffer of server + worker operational log records; backs `GET /api/v1/diagnostics/logs` and the `diagnostics` WebSocket subject (see [`docs/observability.md`](observability.md)) | +| Logging | `internal/log` | `slog` setup plus the fan-out `Handler`/`Sink` (`NewWithSink`) that tees every record to stderr *and* to a sink — the diag buffer on the server, the `worker.diag.` publisher on a worker | +| Products | `internal/product` | Product/preset catalog above OpenJD: embedded built-ins overlaid on stored `custom`/`installed` products | | OpenJD | `internal/openjd` | Template parser, validator, parameter-space expansion | | Worker protocol | `internal/worker/protocol` | Shared worker wire-protocol types (the rest of `internal/worker` is the sqi-worker binary; the server-side status/log ingestion lives in `internal/scheduler`) | | Config | `internal/config` | Typed config struct, layered loader (defaults → file → env → flags) | @@ -80,32 +84,41 @@ main() └─ cobra: serve subcommand 1. Load and validate configuration (config.Load + config.Validate) 2. Initialize slog structured logger - 3. Open SQLite, run pending migrations - 4. Start embedded NATS JetStream server - 5. Create in-process NATS client (internal/bus) - 6. Create Store (internal/store/sqlite) - 7. Create Scheduler (internal/scheduler) — starts goroutine pool - 8. Create WebSocket hub (internal/ws) - 9. Wire auth (internal/server wireAuthDeps) — skipped to the anonymous + 3. Open SQLite, run pending migrations (internal/store/sqlite) and + register it as the "sqlite" readiness checker (Store.Ping, read pool) + 4. Start the background WAL checkpointer (store.checkpoint_interval) + 5. Start the expired-session sweeper (no-op when auth is disabled) + 6. Seed a default farm + queue on first start (no-op once any farm exists) + 7. Start embedded NATS JetStream server, provision streams, and register + it as the "nats" readiness checker + 8. Create in-process NATS client (internal/bus) + 9. Create WebSocket hub (internal/ws) — before the scheduler, so it can be + passed in as the notifier — then wire the diagnostic buffer's notify + callback to it + 10. Create and run Scheduler (internal/scheduler). Scheduler.Run is what + registers every NATS consumer: worker registration/heartbeat/deregister, + task status, task logs, the core-NATS worker.diag.> subscriber, and the + core-NATS work.lease.> request/reply subscriber; it also starts the + heartbeat sweep + 11. Wire auth (internal/server wireAuthDeps) — skipped to the anonymous superuser when auth.enabled is false, so auth-off boot is unchanged: a. Bootstrap the first admin account (no-op once any user exists) b. Select the authenticator chain (API key → session cookie) c. Build the LDAP verifier (if auth.ldap.enabled) d. Build the OIDC provider (if auth.oidc.enabled; issuer discovery is lazy — a brief provider outage must not block boot) - 10. Build chi router, mount middleware and route handlers - 11. Register NATS consumers (worker registration, heartbeat, status, logs) - 12. Start mDNS responder (if discovery.enabled) + 12. Build chi router, mount middleware and route handlers 13. Start HTTP server - 14. Block on SIGINT / SIGTERM - 15. Graceful shutdown: - a. Stop accepting new HTTP connections - b. Drain in-flight HTTP requests + 14. Start mDNS responder (if discovery.enabled) + 15. Block on SIGINT / SIGTERM + 16. Graceful shutdown (30 s deadline, server.ShutdownTimeout): + a. Stop the mDNS responder (goodbye packets first) + b. Stop accepting new HTTP connections and drain in-flight requests c. Stop Scheduler d. Drain NATS in-flight messages, flush JetStream e. Close NATS server - f. Run final SQLite WAL checkpoint - g. Close SQLite + f. Final WAL checkpoint in TRUNCATE mode (checkpointer goroutine, on + context cancel) then close both SQLite pools ``` --- @@ -248,11 +261,13 @@ request/reply). When a request arrives the server: ``` handleLeaseRequest(queueID, workerID) │ - ├─ store.CommittedCores(workerID) → committed (Σ required_cores of assigned+running tasks) + ├─ store.CommittedCores(workerID, worker.CPUCount) → committed (Σ required_cores of assigned+running tasks) ├─ free = worker.CPUCount − committed │ If free ≤ 0: park request in the per-queue waiter registry (~30 s hold) │ - ├─ store.ListReadyTasks(queue, …) → candidates (priority-ordered) + ├─ store.ListReadyTasks(farmID, now, batchSize) → candidates, farm-wide, ordered + │ job priority DESC, job created_at ASC, step order ASC, task created_at ASC + │ (queue affinity is applied per-candidate by WorkerEligible, not by this query) │ │ First-fit walk over candidates: ├─ WorkerEligible(task, worker) → bool (capability/queue/farm/location/amounts match) @@ -265,8 +280,12 @@ handleLeaseRequest(queueID, workerID) │ Decrement free; add to batch │ └─ bus.Reply(batch []AssignMsg) - Each AssignMsg includes: resolved command, args, env, path map, session_id, - isolation identity (username only — see docs/auth.md#task-isolation) + Each AssignMsg includes: task/job/step/attempt IDs, resolved OnRun action + (command, args, timeout), embedded files, ordered environments, job and + task parameters, path map + path deliveries, compute location, and the + isolation identity (username only — see docs/auth.md#task-isolation). + It does NOT carry a session_id: the session is created worker-side and + travels back on the task-status message. ``` A parked request is woken when new work becomes available for that queue (job @@ -320,7 +339,7 @@ NATS consumer (internal/scheduler/taskstatus.go) ├─ Receive task.status message { …, message } ← worker's human-readable reason, if any ├─ store.UpdateTaskAttempt(attempt_id, status, exit_code, end_time, message) ├─ If terminal (succeeded/failed/canceled): - │ store.TransitionTask(task_id, status) + │ store.UpdateTaskStatus(task_id, status) ← state-machine guarded │ If failed/canceled: store.SetTaskFailureReason(task_id, reason) ← see below │ usagePool.ReleaseClaim(claim_id) │ checkStepCompletion → propagateStepDependencies ← marks successor tasks ready @@ -401,9 +420,10 @@ operator-facing view. ``` NATS consumer (internal/scheduler/logingest.go) │ - ├─ Receive task.logs message (chunk: seq, timestamp, data) - ├─ store.InsertLogChunk(attempt_id, seq, timestamp, data) - └─ notifier.NotifyLog(task_id, chunk) ← triggers WebSocket fanout for live tail + ├─ Receive task.logs message on a JetStream push-consumer over SQI_LOGS + ├─ store.CreateTaskLog(store.TaskLog{…}) ← the NATS stream sequence is + │ persisted as the chunk's pagination cursor for the logs REST endpoint + └─ notifier.NotifyLog(task_id, chunk) ← triggers WebSocket fanout for live tail ``` ### 7. Real-time delivery to clients @@ -458,6 +478,26 @@ heartbeat-sweep tick that handles offline-worker cleanup, controlled by └──────────┘ └──────────┘ └──────────────┘ ``` +The diagram shows the happy path only. The complete permitted set +(`internal/store/statemachine.go`) is: + +| From | To | When | +|---|---|---| +| `pending` | `ready` | dependency resolution: all dependency steps completed | +| `pending` | `canceled` | job canceled before step dependencies were satisfied | +| `ready` | `assigned` | scheduler leases the task to a worker | +| `ready` | `canceled` | task canceled while waiting for a worker | +| `assigned` | `running` | worker confirms execution started | +| `assigned` | `ready` | reclaim: assigned worker disconnected or the stale-assigned reaper fired | +| `assigned` | `canceled` | task canceled after assignment, before confirmation | +| `assigned` | `succeeded` / `failed` | the worker's `running` publish was lost (see below) | +| `running` | `succeeded` | worker reports clean exit (exit code 0) | +| `running` | `failed` | worker reports non-zero exit or a fatal error | +| `running` | `ready` | reclaim (worker unreachable) or auto-retry re-queue | +| `running` | `canceled` | task canceled while executing | + +`succeeded`, `failed`, and `canceled` are terminal — no outgoing transitions. + Transitions are validated by `store.ValidateTaskTransition` (`internal/store/statemachine.go`) and enforced by `UpdateTaskStatus` in both store implementations: the SQLite store reads the current status and writes the @@ -466,6 +506,16 @@ writer, and the in-memory fake does the same under its mutex. A transition outside the permitted set returns `store.ErrInvalidTransition` and leaves the row unchanged. +**There are two state machines, in two packages, with two sentinel errors.** +The **task** machine is `store.ValidateTaskTransition` / +`store.ErrInvalidTransition` (`internal/store/statemachine.go`), enforced by +`UpdateTaskStatus` on every write. The **step** machine is +`openjd.ValidateStepTransition` / `openjd.ErrInvalidTransition` +(`internal/openjd/statemachine.go`). The task machine lives in `store` and not +in `openjd` for a hard reason: `openjd` imports `store`, so `store` can never +import `openjd` back. Do not merge the two sentinels — `errors.Is` against the +wrong one silently stops matching. + Two rules keep enforcement safe given that task status arrives over JetStream (at-least-once delivery): @@ -578,12 +628,17 @@ in the first place. | Subject pattern | Transport | Direction | Purpose | |---|---|---|---| | `work.lease.` | Core NATS request/reply | worker → server (request); server → worker (reply) | Worker requests a batch of tasks; server replies with assignments or empty on timeout | -| `task.status.` | JetStream (`TASK_STATUS`) | worker → server | Terminal and intermediate status updates | -| `task.logs.` | JetStream (`TASK_LOGS`) | worker → server | Log chunk delivery | -| `worker.heartbeat` | JetStream (`WORKER_HB`) | worker → server | Liveness heartbeat | -| `worker.register` | JetStream (`WORKER_REG`) | worker → server | Registration at startup | +| `task.status.` | JetStream (`SQI_TASK`, MaxAge 24 h) | worker → server | Terminal and intermediate status updates | +| `task.logs.` | JetStream (`SQI_LOGS`, MaxAge 96 h) | worker → server | Log chunk delivery | +| `task.cancel.` | JetStream (`SQI_CANCEL`, MaxAge 5 min) | server → worker | Cancellation signal; the worker holding the task interrupts the process | +| `worker.register` | JetStream (`SQI_WORKER`, MaxAge 2 min) | worker → server | Registration at startup and on reconnect | +| `worker.heartbeat` | JetStream (`SQI_WORKER`, MaxAge 2 min) | worker → server | Liveness heartbeat | +| `worker.deregister` | JetStream (`SQI_WORKER`, MaxAge 2 min) | worker → server | Graceful departure; marks the worker offline without waiting for heartbeat timeout | | `worker.diag.` | Core NATS (best-effort) | worker → server | Diagnostic log records | +A queue-unaffiliated worker leases on the reserved leaf `work.lease._any` +(`bus.WildcardQueueToken`). + JetStream streams use file-backed storage with configurable size limits. `work.lease.` uses core NATS request/reply — no stream is created for it. The server holds an unfulfillable request in memory for up to 30 s before diff --git a/docs/auth.md b/docs/auth.md index ce511e32..577b1b48 100644 --- a/docs/auth.md +++ b/docs/auth.md @@ -11,11 +11,12 @@ The single switch is `auth.enabled` (config file `auth.enabled`, env As of component A1, the gate is live: flipping `auth.enabled` to `true` and restarting actually locks the server down. Every REST request and the -WebSocket upgrade now require a valid session (see below); there is no more -"scaffolding only" caveat. See [Local accounts](#local-accounts) and -[Login & sessions](#login--sessions) for what that means in practice, and -[First-admin bootstrap](#first-admin-bootstrap) for how to get your first -credential. +WebSocket upgrade now require a valid credential — a session cookie or a +Bearer API key (`auth.Chain(apikey, session)`); there is no more +"scaffolding only" caveat. See [Local accounts](#local-accounts), +[Login & sessions](#login--sessions) and [API keys](#api-keys) for what that +means in practice, and [First-admin bootstrap](#first-admin-bootstrap) for +how to get your first credential. ## Model @@ -53,6 +54,22 @@ The REST resource routes are gated by the auth middleware; the WebSocket upgrade is gated by its own hook; the health/readiness/metrics probes and the OpenAPI spec are always public. +**`auth.enabled` does not gate `/debug/pprof/`.** The Go runtime profiling +endpoints are mounted on the root router, outside `/api/v1` entirely +(`internal/api/router.go`), so the auth middleware, the CSRF guard and the +rate limiter all miss them whether auth is on or off. They are opt-in +(`http.enable_pprof`, default `false`) and the server logs a `WARN` at boot +when they are enabled. Treat that warning as literal. A pprof profile does not +dump memory contents — it will not hand out session tokens, the LDAP bind +password or the OIDC client secret — but it does expose internal code paths, +allocation sites, live goroutine stacks, build and module metadata, and the +process command line to anyone who can reach the port. Worse, because these +routes sit outside the rate limiter, `/debug/pprof/profile?seconds=N` and +`/debug/pprof/trace?seconds=N` accept an arbitrary duration from an +unauthenticated caller, which is a straightforward availability lever. Enable +pprof only on a network you would already trust with an auth-off deployment, +or put a reverse proxy in front of `/debug/`. + ## Local accounts A local account (`internal/store` `User`) has a username (case-insensitive @@ -80,7 +97,8 @@ bad password. Deleting a user cascades to its sessions — see ## Roles & permissions -As of component B1, roles are enforced on every route. There are four +As of component B1, every mutating route and several read routes are gated by a +role→permission policy. There are four built-in roles (no custom-role builder — YAGNI): - **admin** — full access, including user management, API-key management for @@ -88,8 +106,11 @@ built-in roles (no custom-role builder — YAGNI): - **operator** — runs the farm: all jobs, workers, farm infrastructure (farms/queues/storage/compute/usage-pools), products/presets, and diagnostics (server log). -- **user** — submit and control jobs; manage their own API keys; read-only on - infrastructure. +- **user** — submit and control **their own** jobs; manage their own API + keys; read-only on infrastructure. This is the only role without + `jobs.read.all`, so it neither sees nor may mutate a job owned by someone + else — note the counter-intuitive consequence that `read-only` sees every + job while `user` sees only its own. - **read-only** — reads the operational surface; no mutations anywhere; cannot see diagnostics or the user list — but *can* manage its own API keys. @@ -137,7 +158,7 @@ sidebar identity control. | Route | Effect | |---|---| -| `PATCH /api/v1/auth/me` | Sets `display_name`. Returns the same principal shape as `GET /auth/me`. | +| `PATCH /api/v1/auth/me` | Sets `display_name`. Returns the same principal shape as `GET /auth/me`. A body omitting `display_name` is a successful no-op returning the caller's identity unchanged, not an error. | | `PUT /api/v1/auth/password` | Verifies the current password, then sets the new one. | Three choices worth knowing: @@ -181,7 +202,10 @@ An owner naming no known user is rejected with 400 when WebSocket delivery is scoped the same way as REST. Per-job subjects (`jobs/{id}/tasks`, `tasks/{id}/logs`) are authorized once at subscribe time; the global `jobs` subject is filtered per event. A client that cannot resolve a -job's owner receives nothing for it rather than everything. +job's owner receives nothing for it rather than everything. The `diagnostics` +subject is gated separately at subscribe time on `diagnostics.read`, so a +`read-only` or `user` principal is refused the server-log feed on the socket +exactly as it is on `GET /api/v1/diagnostics/logs`. ## Task isolation @@ -262,8 +286,15 @@ Independent of, and in addition to, the supplementary-group stripping below, silently narrowing them: - **`run_as_user` naming a known-privileged account** — `root`, - `Administrator`, `SYSTEM`, and similar — by name, and any account whose - **uid is 0**, regardless of name. + `Administrator`, `Administrators`, `SYSTEM`, `LocalSystem`, + `NETWORK SERVICE`, and `LOCAL SERVICE` — by name, and any account whose + **uid is 0**, regardless of name. Name matching is case-insensitive and + first strips a `DOMAIN\` / `HOST\` / `.\` qualifier and a trailing + `@domain` UPN suffix, so `.\Administrator`, `CORP\Administrator` and + `Administrator@corp.example.com` are all the same refused name. The list + is not configurable: an operator-overridable version would defeat the + point, since the whole risk is a queue pointing at an account more + privileged than the daemon. - **`run_as_group` naming a known-privileged group** — `root`, `wheel`, `admin`, `sudo`, `sudoers`, `adm`, `docker`, `disk`, `shadow`, `staff`, `administrators` — by name. This is a check against the group **you diff --git a/docs/compute-locations.md b/docs/compute-locations.md index 4a93896b..657ed362 100644 --- a/docs/compute-locations.md +++ b/docs/compute-locations.md @@ -37,7 +37,7 @@ The auto-registration is: database error or a create that races another registration) is logged and silently ignored; it never blocks or fails the worker registration itself. -This behaviour is implemented in `ensureComputeLocation` in +This behavior is implemented in `ensureComputeLocation` in `internal/scheduler/scheduler.go`, called on every successful worker registration. diff --git a/docs/configuration.md b/docs/configuration.md index 71a05169..22bf51ad 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,11 +4,18 @@ layers overriding earlier ones: 1. **Built-in defaults** — sensible values for local development. -2. **Config file** — YAML or JSON; searched in `./config/sqi-server.yaml`, - `~/.sqi/sqi-server.yaml`, and `/etc/sqi/sqi-server.yaml` by default. Pass - an explicit path with `--config /path/to/file`. +2. **Config file** — YAML or JSON. With no `--config`, six paths are tried in + order and the **first that exists** is the only one read: + `./config/sqi-server.yaml`, `./config/sqi-server.json`, + `~/.sqi/sqi-server.yaml`, `~/.sqi/sqi-server.json`, + `/etc/sqi/sqi-server.yaml`, `/etc/sqi/sqi-server.json`. Pass an explicit + path with `--config /path/to/file` (a path that does not exist is an + error, unlike the search). 3. **Environment variables** — prefixed `SQI_`, e.g. `SQI_HTTP_ADDR`. -4. **CLI flags** — highest priority; available on the `serve` subcommand. +4. **CLI flags** — highest priority. `--config`, `--log-level` and + `--log-format` are available on every subcommand; the remaining flags + (`--http-addr`, `--http-cors-origins`, `--openjd-enforce-limits`, + `--auth-enabled`, `--auth-validate-job-owner`) are on `serve`. Print the effective merged configuration at any time with: @@ -16,8 +23,10 @@ Print the effective merged configuration at any time with: sqi-server config print ``` -A fully commented example file is at -[`config/sqi-server.example.yaml`](https://github.com/uberware/sqi/blob/main/config/sqi-server.example.yaml). +A commented example file is at +[`config/sqi-server.example.yaml`](https://github.com/uberware/sqi/blob/main/config/sqi-server.example.yaml) +(it does not yet include the `auth` block or `http.cors_origins` — see those +sections below). Duration values use Go syntax: `30s`, `1m30s`, `500ms`, `2h`, etc. @@ -37,6 +46,11 @@ Duration values use Go syntax: `30s`, `1m30s`, `500ms`, `2h`, etc. TCP address the HTTP server listens on. Use `127.0.0.1:8080` to restrict to loopback only. +Must be `host:port`. A host given as a name rather than an IP literal is +resolved at startup, so an unresolvable hostname fails config validation and +the server does not start; `0.0.0.0`, `::` and an empty host are accepted +without resolution. The same rule applies to [`nats.addr`](#natsaddr). + ```yaml http: addr: "0.0.0.0:8080" @@ -79,14 +93,19 @@ Browser origins the CORS middleware allows. Only relevant to a same-origin deployment (where `sqi-server` serves the embedded UI itself) needs none of this. -Each entry must be `scheme://host[:port]`, or the wildcard `"*"`. A trailing -slash, a path, a query, a fragment, or embedded whitespace is rejected at -startup with an `http.cors_origins` validation error — go-chi/cors could -never match such a value, so a typo fails loudly at boot rather than -silently at request time. - -**With `auth.enabled=true` a wildcard is dropped at startup** (and an error -is logged): browsers reject `Access-Control-Allow-Credentials` combined with +Each entry must be `scheme://host[:port]`, or the bare wildcard `"*"`. A +trailing slash, a path, a query, a fragment, or embedded whitespace is +rejected at startup with an `http.cors_origins` validation error — go-chi/cors +could never match such a value, so a typo fails loudly at boot rather than +silently at request time. **A wildcard *pattern* is rejected too** +(`https://*.example.com`, `https://app.example.com*`): go-chi/cors would honor +any embedded `*` as a prefix/suffix match — with credentials, once auth is +enabled — so only the bare `"*"` is supported. Name explicit origins. + +**With `auth.enabled=true` a wildcard is dropped at startup** (and a warning is +logged — or, when the wildcard came from the empty-list default rather than +explicit config, an informational line): browsers reject +`Access-Control-Allow-Credentials` combined with `*`. An empty list defaults to `["*"]` and so is dropped too — meaning a separately-hosted UI must name its origin explicitly here for credentialed cross-origin requests to work at all. See @@ -189,6 +208,12 @@ store: sqlite_path: "/var/lib/sqi/sqi.db" ``` +> **The `migrate` and `backup` subcommands do not read this key.** Their +> `--db` flag defaults to `$SQI_SQLITE_PATH` (note: *not* +> `SQI_STORE_SQLITE_PATH`), falling back to `sqi.db` in the working +> directory. Pass `--db` explicitly, or export both variables, so schema +> migrations and backups operate on the database the server actually uses. + --- ### `store.checkpoint_interval` @@ -638,7 +663,7 @@ four can answer `503` for a deadline. The preset routes — `GET /api/v1/presets/{name}` and `POST /api/v1/presets/{name}/install` — validate a definition too. Its body is -sha256-pinned against the index at `products.preset_library_url`, so it is +sha256-pinned against the index at `preset_library.url`, so it is operator-vouched rather than client-chosen; since EXPR sub-project H1's whole-branch review they nevertheless run under these four keys and under the deadline, because the limits are operator configuration (an operator who @@ -1187,8 +1212,9 @@ superuser. As of component A1, this is a live gate: setting it to `true` requires every REST request and the WebSocket upgrade to carry a valid session, backed by local accounts (see below and [`docs/auth.md`](auth.md)). Role-based -authorization is not enforced yet (component B1) — see the interim gap -documented in [`docs/auth.md`](auth.md#local-accounts). +authorization is enforced as of component B1: every mutating route and several +read routes are gated by a role→permission policy — see +[`docs/auth.md`](auth.md). ```yaml auth: @@ -1321,8 +1347,7 @@ auth: ``` See [`docs/auth.md`](auth.md) for the full authentication model, the local -account model, and the interim authorization gap before role enforcement -(component B1). +account model, and the role/permission matrix enforced on every route. --- @@ -1724,8 +1749,9 @@ for the keys themselves. ### Capability auto-detection (`capabilities.detect` / `capabilities.disable`) Controls the worker's software capability auto-detection: built-in detectors -for Maya, Nuke, Houdini, and Blender run automatically at startup and -advertise a tag (e.g. `maya`, plus `maya-2025`) with value `"true"`, with no +for Maya, Nuke, Houdini, Blender, FFmpeg and SGO Mistika (Boutique/Ultima, VR +and Workflows) run automatically at startup and advertise a tag (e.g. `maya`, +plus `maya-2025`) with value `"true"`, with no per-worker configuration — enough on its own to satisfy the reference DCC products/presets' `key=true` requirement. See [`docs/products.md`](products.md) and @@ -1865,8 +1891,10 @@ discovery: ## See also -- [`config/sqi-server.example.yaml`](https://github.com/uberware/sqi/blob/main/config/sqi-server.example.yaml) — Fully - commented example with every option. +- [`config/sqi-server.example.yaml`](https://github.com/uberware/sqi/blob/main/config/sqi-server.example.yaml) — + Commented example covering the HTTP, NATS, store, log, scheduler, discovery, + OpenJD, diagnostics and preset-library sections. The `http.cors_origins` key + and the whole `auth` block are documented in this reference only. - [`docs/architecture.md`](architecture.md) — Component layout and how configuration values are consumed. - [`docs/operations.md`](operations.md) — Install, upgrade, backup, and log rotation. - [`docs/observability.md`](observability.md) — In-UI diagnostics, REST/WS log API, and external log wiring. diff --git a/docs/contributing.md b/docs/contributing.md index 475ba0b8..07d57458 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -17,18 +17,21 @@ We maintain a [roadmap and architecture guide](roadmap.md) organized by developm git clone https://github.com/Uberware/sqi.git cd sqi -# Go development (server and workers) -cd cmd/sqi-server -go mod download -go build +# Install the git hooks (gofumpt, goimports, go vet, golangci-lint, and the +# Conventional Commits check). Requires gofumpt, goimports and golangci-lint +# on your PATH — see docs/development.md for install commands. +make hooks -# Python development (client SDK and DCC submitters) -cd clients/python -pip install -e . +# Go development (server and workers) — builds the web UI bundle first and +# writes both binaries into ./bin/ +make build + +# Python development (client SDK) +make py-install # Web UI development (TypeScript + React) cd web -npm install +npm ci npm run dev ``` @@ -43,7 +46,8 @@ npm run dev - **Go**: Follow [Effective Go](https://golang.org/doc/effective_go). Run `make fmt` (gofumpt + goimports) and `make lint` (golangci-lint) before submitting - **Python**: Follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Use type hints for all function signatures. Format and lint with ruff: `ruff format && ruff check --fix` — this handles style consistency automatically - **TypeScript/React**: Use ESLint and Prettier with the project configuration. Functional components and hooks preferred -- **Commit messages**: Be clear and specific. Reference issue numbers where relevant (e.g., "Fix scheduler race condition in Phase 1 (#42)") +- **SPDX header**: every source file (Go, TypeScript, YAML, Python, shell) must carry `SPDX-License-Identifier: AGPL-3.0-or-later` before any `package`/`import`/module declaration, in the file's own comment syntax. See [`docs/spdx-header.md`](spdx-header.md) for the exact template. +- **Commit messages**: [Conventional Commits](https://www.conventionalcommits.org) format — `type(scope)?: description` — is **enforced** by the `commit-msg` git hook installed by `make hooks`. Valid types: `feat fix docs style refactor test chore build ci perf revert`. Reference issue numbers in the description where relevant (e.g. `fix(scheduler): correct heartbeat timeout calculation (#42)`). `CHANGELOG.md` is generated from these messages by git-cliff, so a non-conforming commit never reaches the changelog. **Testing:** @@ -95,8 +99,8 @@ Conventions: - Mock at the network boundary (the `apiFetch`/query layer), not at the component internals, so tests exercise real component wiring. - New components and hooks ship with tests; coverage is enforced against the - threshold in `web/vite.config.ts`. See the existing `DataTable.test.tsx`, - `StatusBadge.test.tsx`, and the `src/api` / `src/hooks` tests for the patterns. + threshold in `web/vite.config.ts`. See the existing `StatusBadge.test.tsx`, + `Pagination.test.tsx`, and the `src/api` / `src/hooks` tests for the patterns. **API client pattern.** All server access goes through `src/api/` — never call `fetch` directly from a component. @@ -210,8 +214,8 @@ The [ROADMAP.md](roadmap.md) document outlines development phases. Current prior - **Phase 1** (v0.1 — released): Core scheduler, pull-based workers, basic web UI, OpenJD execution - **Phase 2** (v0.2 — released): Product system, preset library, DCC submitters -- **Phase 3** (next): Auth (LDAP, OAuth2), multi-user role model -- **Phase 4** (planned): Production hardening, PostgreSQL, HA, auto-scaling +- **Phase 3** (complete, unreleased): Auth (local accounts, API keys, RBAC, LDAP/AD, OAuth2/OIDC SSO), multi-user role model, and run-as-user task isolation +- **Phase 4** (next): Production hardening, PostgreSQL, HA, auto-scaling Code contributions aligned with the current phase are most likely to be accepted quickly. Contributions targeting later phases are welcome but may take longer to review if they require design discussion. diff --git a/docs/cross-platform-validation.md b/docs/cross-platform-validation.md index ed43f012..928b76cf 100644 --- a/docs/cross-platform-validation.md +++ b/docs/cross-platform-validation.md @@ -5,10 +5,10 @@ This guide is the manual procedure for confirming `sqi` works on **Linux, macOS, and Windows**. It closes the Phase 1 verification items that CI cannot fully prove on its own: -- **Cross-platform runtime** (`phase1-tests.md` §3): the cross-compiled binaries - actually *start and register*, not just compile. -- **Cross-browser web UI** (`phase1-tests.md` §4): the embedded UI works in - Chromium, Firefox, and WebKit. +- **Cross-platform runtime:** the cross-compiled binaries actually *start and + register*, not just compile. +- **Cross-browser web UI:** the embedded UI works in Chromium, Firefox, and + WebKit. CI on the default Linux runner already exercises the unit suites, the Docker smoke test, and (where wired) the Playwright suite. The steps below are what you @@ -145,7 +145,7 @@ npx playwright test --project=firefox npx playwright test --project=webkit ``` -Expected: `6 passed` (2 specs × 3 engines). An HTML report lands in +Expected: `6 passed` (2 tests × 3 engines). An HTML report lands in `web/playwright-report/` (open with `npx playwright show-report`). The suite's global setup builds the binaries if `bin/sqi-server`/`bin/sqi-worker` @@ -164,8 +164,8 @@ manage process lifecycle. # Git Bash — terminal 1: start the server (serves the embedded UI on :8080) bin/sqi-server.exe serve -# terminal 2: start a worker (auto-discovers the server via mDNS, or pass --nats-url) -bin/sqi-worker.exe start --allow-root # --allow-root only if running elevated +# terminal 2: start a worker (auto-discovers the server via mDNS, or set SQI_WORKER_NATS_URL) +bin/sqi-worker.exe start # no allow-root needed on Windows — the root check is POSIX-only # terminal 3: run Playwright against the running server cd web @@ -192,11 +192,11 @@ npx playwright install-deps ## C. Manual cross-browser walk-through (sign-off) -For the human cross-browser sign-off (`phase1-tests.md` §4), in **each** of +For the human cross-browser sign-off, in **each** of Chrome/Edge, Firefox, and Safari (Safari on macOS only): 1. Start the stack: `make run-server` (terminal 1) and - `bin/sqi-worker start` (terminal 2). Open `http://localhost:8080/`. + `./bin/sqi-worker start` (terminal 2, after `make build-worker`). Open `http://localhost:8080/`. 2. **Dashboard** loads; the connection badge is green (WebSocket connected); the seeded/started worker shows online. 3. **Submit** (`/submit`): pick a queue, use **Load example → single-step shell @@ -226,9 +226,10 @@ Chrome/Edge, Firefox, and Safari (Safari on macOS only): - **macOS AppleDouble files.** On non-APFS volumes macOS writes `._*` sidecars; the tooling already filters them (Playwright `testIgnore`, Vitest exclude, the embed filter). Harmless on Linux/Windows. -- **`--allow-root`.** The worker refuses to run as root unless `--allow-root` - (or `SQI_WORKER_ALLOW_ROOT=true`) is set — relevant when validating in a - container or as Administrator/root in a VM. +- **Running as root/Administrator.** The worker refuses to run as root on + Linux/macOS unless `worker.allow_root: true` is set in the worker config or + `SQI_WORKER_ALLOW_ROOT=true` is exported. There is no `--allow-root` flag — + relevant when validating in a container or as root in a VM. --- diff --git a/docs/dcc-submitters.md b/docs/dcc-submitters.md index 9facebe9..fff3e30a 100644 --- a/docs/dcc-submitters.md +++ b/docs/dcc-submitters.md @@ -39,7 +39,7 @@ lets the artist pick one, fetches its parameters (`GET exactly what the web `/submit` page does. Everything DCC-specific is scene/context extraction (a scene path, a frame range, which render target is selected) that pre-fills matching parameters by **name**, never by which -product was chosen. See [Reference presets](#reference-presets) for the six +product was chosen. See [Reference presets](#reference-presets) for the nine products this repo ships to give the framework something real to target, and [The parameter convention contract](#the-parameter-convention-contract) for exactly how pre-fill works. @@ -311,11 +311,11 @@ worker: - maya=true ``` -(substitute `houdini=true`, `nuke=true`, or `blender=true` for the other -three presets), or add a [custom -detector](worker-capabilities.md#writing-custom-detectors) instead. A job -submitted against one of these products sits `ready` forever if no worker -ends up advertising the tag (auto-detected or manual). +(substitute `houdini=true`, `nuke=true`, `blender=true`, `mistika=true`, +`mistikavr=true`, or `mistikaworkflows=true` for the other seven presets), or +add a [custom detector](worker-capabilities.md#writing-custom-detectors) +instead. A job submitted against one of these products sits `ready` forever if +no worker ends up advertising the tag (auto-detected or manual). --- @@ -564,8 +564,8 @@ for the four existing examples): Real-DCC verification (Maya, Houdini, Nuke) is manual — there's no CI runner for licensed software. Blender is covered by an automated integration smoke test (see `clients/submitter/tests/integration/test_e2e.py`, -`SQI_TEST_BLENDER=1`) since it's freely installable, but should still be spot --checked manually after any change to its adapter/add-on. +`SQI_TEST_BLENDER=1`) since it's freely installable, but should still be +spot-checked manually after any change to its adapter/add-on. For each host, after installing per [Installation per host](#installation-per-host): @@ -619,7 +619,7 @@ treat it, not this doc, as the source of truth if they ever disagree. The same applies: a JS submitter that wants "pre-fill from the host app" pre-fills by matching the same parameter names, with the same case/separator-insensitive rules and the same additive-only guarantee. No Qt, no Python, and none of -`sqi-submitter`'s packaging is required to build a fully-functional submitter +`sqi-submitter`'s packaging is required to build a fully functional submitter against this contract — only an HTTP client. --- diff --git a/docs/development.md b/docs/development.md index c6e908bc..5b5cb243 100644 --- a/docs/development.md +++ b/docs/development.md @@ -68,16 +68,24 @@ Run `make` (no arguments) to see all available targets with descriptions. | `make test-ldap` | Run the LDAP tests against a real OpenLDAP directory in a container (needs Docker; **skips** without it) | | `make test-oidc` | Run the SSO tests against a real Keycloak in a container (needs Docker; **skips** without it) | | `make test-isolation` | Run run-as-user task-isolation tests as real root against real OS accounts in a container (needs Docker; **skips** without it) | +| `make test-isolation-windows` | Run the Windows run-as-user isolation tests against real local accounts — must be run from an **elevated** shell on a real Windows host (no container); exits 0 with a message when not elevated | +| `make test-conformance` | Run the official OpenJD conformance suite against the vendored `third_party/` fixtures (build tag `conformance`) | | `make test-expr-oracle` | Differential-test the EXPR evaluator against the OpenJD reference implementation (needs `python3`; **skips** without it) | | `make test-preset-library` | Validate the **published** preset library against the validator in your tree (needs network; **skips** when the library is unreachable, **fails** when it is reachable but invalid) | | `make expr-oracle-venv` | Create `.venv-oracle/` with the pinned reference implementation (`make test-expr-oracle` does this on demand) | | `make smoke` | End-to-end smoke test against the real binaries (REST + WebSocket) | +| `make auth-demo` | Run the auth-surface demo against a live local farm (`KEEP=1` leaves it running) | | `make bench` | Run benchmarks | | `make lint` | Run `golangci-lint` | | `make lint-fix` | Run `golangci-lint --fix` | +| `make lint-actions` | Lint the GitHub Actions workflows with `actionlint` (via `go run`; no install needed) | | `make fmt` | Format all Go files with `gofumpt` and `goimports` | -| `make vet` | Run `go vet ./...` | +| `make fmt-check` | Check formatting without modifying files (the `make ci` gate) | +| `make vet` | Run `go vet` over the filtered package list (excludes `web/node_modules/`) | | `make docs` | Serve Go package docs at `localhost:8080` via `pkgsite` | +| `make docs-site-install` | Create `.venv-docs/` and install the pinned MkDocs dependencies | +| `make docs-site` | Build the MkDocs documentation site with `--strict` (the CI gate) | +| `make docs-site-serve` | Serve the documentation site locally with live reload | | `make changelog` | Regenerate `CHANGELOG.md` from Conventional Commits via `git-cliff` (`VERSION=x.y.z` tags the pending release) | | `make hooks` | Install git hooks via `lefthook` | | `make clean` | Remove build artifacts and `coverage.out` | @@ -271,7 +279,7 @@ on the 0.x line, where breaking changes are permitted in minor bumps. The specification outranks it. When the two disagree, read `third_party/openjd-specifications/` and decide; do not change sqi to match the reference. **Most** baselined entries are cases where **the reference is -wrong** — the corpus currently scores 891/1052 agreeing with 161 baselined +wrong** — the corpus currently scores 930/1063 agreeing with 133 baselined divergences, and each one's reasoning is argued in `test/oracle/baseline.txt`, which is the authority on any individual ruling (an earlier revision of this paragraph said "three of the five", a count that went stale several waves ago @@ -313,14 +321,18 @@ sqi/ │ └── sqi-worker/ Worker entry point ├── internal/ │ ├── api/ HTTP router, REST handlers, WebSocket upgrade, OpenAPI spec +│ ├── auth/ Auth gate, accounts, sessions, API keys, RBAC policy, LDAP, OIDC │ ├── bus/ Typed NATS JetStream client wrapper │ ├── config/ Typed config struct, layered loader +│ ├── diag/ In-memory diagnostic-log ring buffer │ ├── discovery/ mDNS _sqi._tcp responder │ ├── health/ /healthz and /readyz handlers │ ├── log/ slog helpers │ ├── metrics/ Prometheus metric definitions -│ ├── middleware/ HTTP middleware (logging, metrics, versioning) +│ ├── middleware/ HTTP middleware (logging, metrics, versioning) │ ├── openjd/ OpenJD parser, validator, parameter-space expansion +│ ├── presetgen/ Generates the preset-library index.json + definition files +│ ├── product/ Products/presets — the catalog layer over OpenJD templates │ ├── scheduler/ Assignment loop, worker registry, heartbeat sweep │ ├── server/ Process boot, graceful shutdown orchestration │ ├── store/ Store interface + SQLite implementation + migrations @@ -330,6 +342,9 @@ sqi/ │ └── ws/ WebSocket hub, subscription management, scheduler-driven fanout ├── pkg/ Public Go API (currently empty; see pkg/doc.go) ├── api/ Source-of-truth specs: OpenAPI 3.1, JSON schemas +├── clients/ Python packages: clients/python (sqi-sdk), clients/submitter (sqi-submitter) +├── presets/ Reference (presets/sqi) and test (presets/testing) preset YAML +├── third_party/ Vendored OpenJD specifications submodule (specs, RFCs, and conformance fixtures) ├── web/ Frontend source; web/dist is embedded ├── config/ Example config files ├── deploy/ Docker and infrastructure manifests @@ -340,12 +355,16 @@ sqi/ ### Key conventions -**No cross-imports between internal packages at the same level.** The -dependency direction is: +**Imports run one way down the dependency direction.** Composition flows: `cmd` → `internal/server` → `internal/api`, `internal/scheduler` → `internal/store`, `internal/bus` +Higher-level packages import lower-level ones — `internal/api`, for instance, +imports `internal/auth`, `internal/openjd`, `internal/scheduler`, `internal/store`, +`internal/ws` and more — but never the reverse: nothing a package depends on may +depend back on it. -**Interfaces over concrete types at package boundaries.** Handlers receive the -`store.Store` interface, not `*sqlite.Store`, so tests can inject a fake. +**Interfaces over concrete types at package boundaries.** Handlers and the +scheduler receive the `store.Store` interface, never `*sqlite.Store`, so tests +can inject a fake. **One file per route group.** `internal/api/jobs.go`, `internal/api/tasks.go`, `internal/api/workers.go`, etc. Each file owns its handler struct, wire-format @@ -486,6 +505,24 @@ statement, and add a corresponding stub to the in-memory fake in > [the durable-failure-reason table](architecture.md#5-status-ingestion) for > every existing path and its reason string. +### Step 3b — Gate it with a permission (if the route is not public) + +A new permission lands in **two** places or it silently half-works: + +1. **Server** — add the permission constant and its role grants to + `internal/auth/policy/policy.go`, then mount the route behind the matching + authorization middleware in `internal/api/router.go`. +2. **Web** — add the same string to the `Permission` union **and** the + `PERMISSION_SET` literal in `web/src/auth/policy.ts`. The + `Record` type on `PERMISSION_SET` makes an omission a + typecheck failure, and `web/src/auth/policy.test.ts` mirrors the server + grants against `ALL_PERMISSIONS`, so a server-only permission fails a test + rather than shipping with no client-side gate. + +Then gate the UI affordance itself with `can(principal, '')` and, +for a whole route, wrap it in `` — see +[`web-development.md`](web-development.md#role-gating-can-requirerole-navcard-filtering). + ### Step 4 — Update the OpenAPI spec Add the new path to `internal/api/openapi.yaml`: @@ -1063,7 +1100,7 @@ Gate command line (from `clients/submitter`, with `sqi-sdk` resolvable — `pip install -e ../python` first if not already installed): ```sh -ruff format --check . && ruff check . && mypy src && pytest -q +ruff format --check . && ruff check . && mypy src && mypy --python-version=3.13 tests && pytest -q ``` `tests/integration/` is skipped unless `SQI_TEST_SERVER_URL` points at a live diff --git a/docs/index.md b/docs/index.md index c8237377..e8b98540 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ `sqi` (pronounced "sky") is an open source distributed task and render farm manager built for modern production pipelines. It is designed to run simply on a handful of local workstations and scale to hybrid on-premises and cloud infrastructure without changing how you work. -> **Status:** v0.2.0 (Phase 2) released. Builds on the Phase 1 core with products and presets as an authoring layer over OpenJD, a community preset-library integration, product-driven submission, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** (opt-in auth & multi-user) — identity complete, task isolation in progress. Contributions, feedback, and discussion welcome. +> **Status:** v0.2.0 (Phase 2) released. Builds on the Phase 1 core with products and presets as an authoring layer over OpenJD, a community preset-library integration, product-driven submission, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** (opt-in auth & multi-user) — local accounts, API keys, RBAC, job-owner identity, LDAP/AD, OAuth2/OIDC SSO, and run-as-user task isolation — is complete and merged on `main`, unreleased. **Phase 4** (production hardening) is next. Contributions, feedback, and discussion welcome. --- @@ -22,7 +22,7 @@ The render farm management space is in an awkward moment. Legacy on-premises sys **Not tied to any cloud provider.** Workers run on Linux, macOS, and Windows — bare metal, VMs, or containers. Cloud compute locations are supported across AWS, GCP, Azure, and any provider that can run a container or a binary. Your control plane runs where you want it. -**OpenJD compatible.** `sqi` adopts the [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications) format as its native job execution layer — not an import step or a partial translation, but the format `sqi` jobs are authored and validated in directly. This is a real standard designed for portability, not a proprietary format. Conformance is measured, not asserted: `sqi` runs the official OpenJD conformance test suite on every CI build and publishes the current pass rate and known gaps at [`openjd-conformance.md`](openjd-conformance.md). +**OpenJD compatible.** `sqi` adopts the [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications) format as its native job execution layer — not an import step or a partial translation, but the format `sqi` jobs are authored and validated in directly. This is a real standard designed for portability, not a proprietary format. Conformance is measured, not asserted: `sqi` runs the official OpenJD conformance test suite on every CI build and publishes the current pass rate and known gaps at [`openjd-conformance.md`](openjd-conformance.md). One caveat is by design rather than a gap: a template that opts into an OpenJD extension `sqi` does not implement is rejected outright, rather than being accepted and misinterpreted. The official `EXPR` (expression language) extension **is** implemented and supported — see [`openjd-extensions/expr.md`](openjd-extensions/expr.md). **General purpose.** Rendering is the primary use case and the domain `sqi` is designed around, but the job model is general. Any workload expressible as a command with defined inputs, outputs, and environment is a valid `sqi` job — simulation, transcoding, machine learning pipelines, data processing, software development, or anything else a studio runs at scale. diff --git a/docs/observability.md b/docs/observability.md index 9ea5d164..660d4c70 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -292,6 +292,14 @@ file forwarding — see [Out-of-band wiring](#out-of-band-wiring)). > **Buffer lifetime:** the ring buffer is **in-memory only** and is lost on > server restart. If you need durable operational logs beyond `buffer_size` > records per component, ship them out-of-band as well. +> +> **Component ceiling:** the buffer also caps the number of distinct component +> rings it retains — **512** by default (`internal/diag`'s +> `defaultMaxComponents`, not currently an operator setting). When the ceiling +> is exceeded the component whose most-recent record is oldest is evicted +> wholesale (LRU by latest activity); the `server` component is never evicted. +> Worst-case buffer memory is therefore `512 × buffer_size` records, and a farm +> with more than 512 workers will see the quietest workers' panels go empty. --- @@ -415,7 +423,7 @@ enabled records to all `diagnostics` subscribers. | `DEBUG` | Verbose internals — scheduler tick details, NATS message counts. Not emitted at the default `info` level. | | `INFO` | Normal operational events — worker registration, task assignment, startup. | | `WARN` | Recoverable problems — missed heartbeat, executor warning, retried operation. | -| `ERROR` | Failures that required operator attention — database errors, NATS disconnection. | +| `ERROR` | Failures that require operator attention — database errors, NATS disconnection. | ### Correlation keys @@ -526,7 +534,7 @@ docker logs -f sqi-server docker logs sqi-server 2>&1 | jq 'select(.level == "WARN" or .level == "ERROR")' ``` -For centralised collection, use the `fluentd` or `gelf` driver: +For centralized collection, use the `fluentd` or `gelf` driver: ```sh docker run -d --name sqi-server \ @@ -658,9 +666,11 @@ Operators should still follow standard practices: - Do not set `SQI_LOG_LEVEL=debug` in production unless actively diagnosing a problem. Debug output is more verbose and may log additional internal state. -- The `GET /api/v1/diagnostics/logs` endpoint sits under the same authorization - as all other sqi read endpoints. Restrict network access to the sqi-server - port accordingly. +- `GET /api/v1/diagnostics/logs` requires the dedicated `diagnostics.read` + permission (`internal/auth/policy`), not the generic read grant, and the + WebSocket `diagnostics` subject is gated by the same permission at + subscribe time. When auth is disabled both are open to any caller that can + reach the port, so restrict network access to the sqi-server port accordingly. - If you forward diagnostic logs to a third-party log aggregator, ensure that aggregator's access controls are appropriate for operational metadata. diff --git a/docs/openjd-conformance.md b/docs/openjd-conformance.md index 7c2e57ab..ee60b7da 100644 --- a/docs/openjd-conformance.md +++ b/docs/openjd-conformance.md @@ -386,6 +386,7 @@ measured results, not assertions: | `base/job_templates` | **449 / 449 pass** | | `base/env_templates` | not applicable — standalone environment templates unsupported (39 tests) | | `TASK_CHUNKING/job_templates` | **11 / 11 pass** | +| `REDACTED_ENV_VARS` | no template fixtures — the suite ships only `jobs/` (8 job-execution tests), which are out of scope; see **Scope** below | | `EXPR/job_templates` | **206 / 209 pass, 3 baselined** — see [EXPR](#expr) | | `EXPR/env_templates` | not applicable — standalone environment templates unsupported (6 tests) | | `FEATURE_BUNDLE_1/job_templates` | not applicable — extension not registered (41 tests) | diff --git a/docs/openjd-extensions.md b/docs/openjd-extensions.md index 6877c7d2..b81e3a18 100644 --- a/docs/openjd-extensions.md +++ b/docs/openjd-extensions.md @@ -60,5 +60,6 @@ What the worker does at runtime, with file references. - [`TASK_CHUNKING`](openjd-extensions/task-chunking.md) — official. - [`REDACTED_ENV_VARS`](openjd-extensions/redacted-env-vars.md) — official. +- [`EXPR`](openjd-extensions/expr.md) — official. - [`SQI_PATH_TRANSLATION`](openjd-extensions/path-translation.md) — vendor. - [`SQI_CHUNK_BOUNDS`](openjd-extensions/sqi-chunk-bounds.md) — vendor. diff --git a/docs/openjd-extensions/expr.md b/docs/openjd-extensions/expr.md index 20373f3b..60fd0f22 100644 --- a/docs/openjd-extensions/expr.md +++ b/docs/openjd-extensions/expr.md @@ -21,9 +21,10 @@ A template declares it with `extensions: [EXPR]`. `presets/sqi/ffmpeg-segment-transcode-*.yaml` are the shipped worked examples of this extension: three variants of the same segmented ffmpeg -transcode, all three of which declare `extensions: [EXPR]`. One, -`ffmpeg-segment-transcode-expr` ("Portable"), builds its ffmpeg concat file -list from an EXPR comprehension instead of a join-time shell script, so it +transcode, all three of which declare `extensions: [EXPR]` +(`presets/sqi/ffmpeg-sequence-encode.yaml` declares it too, more modestly). +One, `ffmpeg-segment-transcode-expr` ("Portable"), builds its ffmpeg concat +file list from an EXPR comprehension instead of a join-time shell script, so it runs on Linux, macOS, and Windows workers alike with no shell at all. That list-building expression references only job parameters, so it is fully resolved at **submission** (phase 2, not phase 3) and its cost is charged @@ -76,6 +77,18 @@ The extension is implemented and tested as follows: and the six path properties (`__property_name__` and friends), which are registered in the same table — `internal/openjd/expr`'s `functionShapes`, whose count `funcs_internal_test.go` pins at 80. +- **`apply_path_mapping` is host-context-only.** It is registered FLAT in + `functionShapes` — the evaluator itself imposes no scope restriction — and + the rule is enforced one layer up, by the phase-2 checker: `hostOnlyFunctions` + (`internal/openjd/exprcheck.go`) walks `Expression.CalledFunctions()` at every + position whose `Scope.IsHostContext()` is false and rejects the call there. + `IsHostContext` is a positive list of exactly three scopes — + `ScopeJobEnvironment`, `ScopeStepEnvironment`, `ScopeStepScript` + (`internal/openjd/scope.go`) — so the function is refused in the job `name` + field, in `hostRequirements`, in `parameterSpace` and in a + `.let` block, all of which are evaluated at submission before + any session exists. The list is positive rather than a negation so that a + scope added later fails CLOSED. - **Bounded evaluation** — per-`Eval` memory and operation limits (`internal/openjd/expr/limits.go`, `meter.go`), plus a per-symbol-table retained-bytes bound on the worker's `let:` evaluator @@ -856,7 +869,13 @@ the configuration guides: differently on purpose. On the **server**, tightening only rejects work at submit, on the one request that can report it, so the floors are sized against this repository's own reference presets (`presets/sqi/*.yaml`) — - whose worst case costs 15 positions — with wide headroom. On the + whose worst case costs 31 positions template-wide, and 390 live bytes with + 15 operations in any single evaluation — those three all set by + `ffmpeg-segment-transcode-expr`, and re-measured on every run by + `TestExprLimits_FloorsAcceptReferencePresets`. Retained bytes are + effectively zero for all fourteen (the binary search floors that dimension + at 1, so a reported "1" means "nothing measurable"). Headroom is wide + throughout (8x to 65536x). On the **worker**, tightening rejects work *after* the job was accepted, so the floors are sized well above what a preset happens to cost: the server's own **defaults** for the two per-evaluation dimensions, and 2,000 positions diff --git a/docs/openjd-extensions/path-translation.md b/docs/openjd-extensions/path-translation.md index 3204778b..5766474a 100644 --- a/docs/openjd-extensions/path-translation.md +++ b/docs/openjd-extensions/path-translation.md @@ -1,4 +1,5 @@ + # SQI_PATH_TRANSLATION - Origin: vendor diff --git a/docs/openjd-extensions/sqi-chunk-bounds.md b/docs/openjd-extensions/sqi-chunk-bounds.md index 6d3014fc..57969a98 100644 --- a/docs/openjd-extensions/sqi-chunk-bounds.md +++ b/docs/openjd-extensions/sqi-chunk-bounds.md @@ -43,6 +43,7 @@ Example: - Every `CHUNK[INT]` parameter must be `CONTIGUOUS` (the default). A `NONCONTIGUOUS` chunk is rejected, because `.Start`/`.End` are undefined across the gaps of a non-contiguous set. + See `internal/openjd/validate.go` (`validateChunkBounds`). Note: `.Start`/`.End` describe the enclosing span of the chunk, so a stepped diff --git a/docs/openjd-extensions/task-chunking.md b/docs/openjd-extensions/task-chunking.md index cf835518..7a8d529e 100644 --- a/docs/openjd-extensions/task-chunking.md +++ b/docs/openjd-extensions/task-chunking.md @@ -18,6 +18,7 @@ A task parameter may use the `CHUNK[INT]` type. The template must declare - Declaring `CHUNK[INT]` without `TASK_CHUNKING` in `extensions` is rejected (`/steps/{i}/parameterSpace`). - `TASK_CHUNKING` may be declared without using `CHUNK[INT]`. + See `internal/openjd/validate.go` (`validateExtensions`). ## Worker behavior diff --git a/docs/openjd-submission.md b/docs/openjd-submission.md index 971b3dc2..8b4edb6d 100644 --- a/docs/openjd-submission.md +++ b/docs/openjd-submission.md @@ -40,7 +40,7 @@ steps: # at least one required taskParameterDefinitions: - name: type: INT | FLOAT | STRING | PATH - range: + range: | [, ...] # string form is INT/CHUNK[INT] only combination: # optional; default is Cartesian product of all params dependencies: # optional; list of step dependencies - dependsOn: # name of a step that must complete first @@ -48,6 +48,12 @@ steps: # at least one required jobEnvironments: [...] # optional; applied to every step ``` +The type lists above are the base spec's. Declaring `extensions: [TASK_CHUNKING]` +adds the `CHUNK[INT]` task-parameter type, and `extensions: [EXPR]` adds RFC 0007's +`BOOL`, `RANGE_EXPR` and six `LIST[*]` job-parameter types along with +case-insensitive type names — see +[`docs/openjd-extensions.md`](openjd-extensions.md). + --- ## 1. Minimal single-task job @@ -220,13 +226,19 @@ parameterSpace: taskParameterDefinitions: - name: InputChunk type: PATH - range: "chunk_001.dat chunk_002.dat chunk_003.dat" + range: ["chunk_001.dat", "chunk_002.dat", "chunk_003.dat"] - name: OutputChunk type: PATH - range: "out_001.dat out_002.dat out_003.dat" + range: ["out_001.dat", "out_002.dat", "out_003.dat"] combination: "(InputChunk, OutputChunk)" # 3 tasks, not 9 ``` +A STRING, PATH or FLOAT `range` **must** be a YAML list. The succinct +`` string form (`"1-100:2"`) exists only for INT and +`CHUNK[INT]`; a bare string given to a STRING, PATH or FLOAT parameter is +decoded into `RangeExpr`, which `expand.go` never reads for those types, so the +template validates and then fails at submission with `range list is empty`. + ### 2d. Mixed combination Zip one pair, then cross it with a third range: @@ -462,8 +474,19 @@ double-brace syntax to reference values: | Expression | Resolves to | |---|---| -| `{{Param.Name}}` | Job-level parameter value | +| `{{Param.Name}}` | Job-level parameter value (a `PATH` parameter arrives path-mapped) | +| `{{RawParam.Name}}` | The same job-level parameter, *without* path mapping applied | | `{{Task.Param.Name}}` | Task-level parameter value (from `parameterSpace`) | +| `{{Task.RawParam.Name}}` | The same task-level parameter, without path mapping applied | +| `{{Task.File.Name}}` | Path of an embedded file materialized for the task (step script scope only) | +| `{{Session.WorkingDirectory}}` | The session's working directory on the worker | +| `{{Session.PathMappingRulesFile}}` | Path of the OpenJD `pathmapping-1.0` rules file, when the session has rules | +| `{{Session.HasPathMappingRules}}` | Whether the session has any path-mapping rules | + +`Session.*` is unavailable in job-level positions (the job `name`, a step's +`hostRequirements` and its `parameterSpace`), which are evaluated at +submission before any session exists. `Task.*` is available only inside a +step's `script`. --- diff --git a/docs/operations.md b/docs/operations.md index 32ce671e..cb1e2d7c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -196,11 +196,11 @@ sqi-server migrate down --config /etc/sqi/sqi-server.yaml sudo systemctl start sqi-server ``` -Always take a backup before upgrading (see [Backup and Restore](#backup-and-restore)). +Always take a backup before upgrading (see [Backup and restore](#backup-and-restore)). --- -## Backup and Restore +## Backup and restore ### Online backup with `sqi-server backup` @@ -278,7 +278,7 @@ worked examples for wiring sqi logs to journald, Docker, Loki, and ELK. ### Output format -By default `sqi-server` writes structured JSON logs to stdout: +By default `sqi-server` writes structured JSON logs to **stderr**: ```json {"time":"2026-01-15T10:00:00.000Z","level":"INFO","msg":"server started","addr":"0.0.0.0:8080"} @@ -314,17 +314,17 @@ journalctl -u sqi-server -f journalctl -u sqi-server -n 1000 -o json ``` -To write to a file instead, redirect stdout in the service unit or use a +To write to a file instead, redirect stderr in the service unit or use a log-forwarding agent (Fluentd, Vector, Promtail) reading from journald. ### Log rotation -Because `sqi-server` writes to stdout rather than a file, log rotation is +Because `sqi-server` writes to stderr rather than a file, log rotation is handled outside the process: - **journald** rotates automatically; tune retention with `journald.conf` (`SystemMaxUse`, `MaxRetentionSec`). -- **File-based logging**: if you redirect stdout to a file, use `logrotate` +- **File-based logging**: if you redirect stderr to a file, use `logrotate` with `copytruncate` (no signal needed — the server does not hold a file descriptor to a log file): ``` @@ -347,21 +347,30 @@ handled outside the process: ### Available metrics -| Metric | Type | Description | -|---|---|---| -| `sqi_http_requests_total` | counter | HTTP requests by method, path, and status | -| `sqi_http_request_duration_seconds` | histogram | HTTP request latency | -| `sqi_scheduler_queue_depth` | gauge | Leasable ready tasks waiting for assignment, by queue (excludes tasks in retry backoff and tasks under paused/parked jobs) | -| `sqi_scheduler_tasks_total` | counter | Tasks processed by final status | -| `sqi_scheduler_assignment_duration_seconds` | histogram | Time from ready → assigned | -| `sqi_scheduler_idle_workers` | gauge | Workers online but not assigned a task | -| `sqi_workers_total` | gauge | Registered workers by status | -| `sqi_nats_published_total` | counter | NATS messages published by subject | -| `sqi_nats_consumed_total` | counter | NATS messages consumed by subject | -| `sqi_db_query_duration_seconds` | histogram | SQLite query latency by operation | -| `sqi_usage_active_claims` | gauge | Active usage-pool claims by pool | -| `sqi_scheduler_task_retries_total` | counter | Tasks re-queued by automatic retry, by queue | -| `sqi_scheduler_jobs_autoparked_total` | counter | Jobs auto-parked at their failure limit, by queue | +| Metric | Type | Labels | Description | +|---|---|---|---| +| `sqi_http_requests_total` | counter | `method`, `path`, `status_code` | HTTP requests by method, path, and status | +| `sqi_http_request_duration_seconds` | histogram | `method`, `path` | HTTP request latency | +| `sqi_scheduler_queue_depth` | gauge | `queue` | Leasable ready tasks waiting for assignment, by queue (excludes tasks in retry backoff and tasks under paused/parked jobs) | +| `sqi_scheduler_tasks_total` | counter | `queue`, `status` | Tasks processed by final status | +| `sqi_scheduler_assignment_duration_seconds` | histogram | `result` | Wall-clock time for a single task-assignment attempt, by `result` (`assigned`, `deferred`, `error`) — not queue residency | +| `sqi_scheduler_idle_workers` | gauge | `farm` | Workers online but not assigned a task | +| `sqi_workers_total` | gauge | `status` | Registered workers by status | +| `sqi_nats_published_total` | counter | `subject` | NATS messages published by subject | +| `sqi_nats_consumed_total` | counter | `subject` | NATS messages consumed by subject | +| `sqi_db_query_duration_seconds` | histogram | `operation` | SQLite query latency by operation | +| `sqi_usage_active_claims` | gauge | `pool` | Active usage-pool claims by pool | +| `sqi_scheduler_task_retries_total` | counter | `queue` | Tasks re-queued by automatic retry, by queue | +| `sqi_scheduler_jobs_autoparked_total` | counter | `queue` | Jobs auto-parked at their failure limit, by queue | + +> **Five of these are registered but not yet populated**, so they emit no +> samples at all — a Prometheus `*Vec` with no children produces no series, and +> an alert written against one will sit silent rather than fire: +> `sqi_scheduler_tasks_total`, `sqi_scheduler_assignment_duration_seconds`, +> `sqi_nats_published_total`, `sqi_nats_consumed_total`, and +> `sqi_db_query_duration_seconds`. (The worker exports its own +> `sqi_worker_nats_published_total` / `sqi_worker_nats_consumed_total` on its +> metrics port; those are different metrics and are populated.) ### Prometheus scrape config @@ -413,17 +422,23 @@ groups: `sqi-server` handles `SIGINT` and `SIGTERM` with a graceful shutdown sequence: -1. Stop accepting new HTTP and WebSocket connections. -2. Wait for in-flight HTTP requests to complete. -3. Drain and flush the embedded NATS JetStream (in-flight messages are - acknowledged or requeued). -4. Run a final WAL checkpoint on the SQLite database. -5. Close all open database connections. -6. Exit with code 0. - -Under systemd the `TimeoutStopSec=60s` in the example unit file gives the -server 60 seconds to drain. Increase this if you have long-running HTTP -streams or large NATS queues. +1. Stop the mDNS responder (goodbye packets first, so discoverers drop the + service immediately instead of waiting for the record to expire). +2. Stop accepting new HTTP and WebSocket connections. +3. Wait for in-flight HTTP requests to complete. +4. Stop the scheduler. +5. Drain and flush the embedded NATS JetStream (in-flight messages are + acknowledged or requeued), then shut down the broker. +6. Run a final WAL checkpoint in TRUNCATE mode on the SQLite database. +7. Close both SQLite connection pools (write and read). +8. Exit with code 0. + +The server imposes its own **30 s** internal drain deadline +(`server.ShutdownTimeout`, a compile-time constant — not a config key); past +that it logs `graceful shutdown timed out after 30s` and exits. The example +unit's `TimeoutStopSec=60s` is deliberate headroom over that 30 s so systemd +never SIGKILLs mid-drain; raising it further has no effect on how long the +server actually waits. --- diff --git a/docs/preset-library.md b/docs/preset-library.md index 73e115dc..f897802d 100644 --- a/docs/preset-library.md +++ b/docs/preset-library.md @@ -118,13 +118,16 @@ A first install returns HTTP 201; an update or reinstall returns HTTP 200. ## REST endpoints -All endpoints return 503 when `preset_library.url` is empty. +All endpoints return 503 when `preset_library.url` is empty. A 503 also means +the definition's validation exceeded `openjd.expr_submission_deadline` on this +server (retry; it is not a rejection of the preset). An unreachable or +unparseable **index** is 502. | Method | Path | Description | |---|---|---| -| `GET` | `/api/v1/presets` | List all presets with per-preset status. `?refresh=true` forces a re-fetch of the index. | -| `GET` | `/api/v1/presets/{name}` | Preview a preset — metadata, template, and status. 404 if not in the index. 422 if the definition cannot be fetched or parsed. | -| `POST` | `/api/v1/presets/{name}/install` | Install or update the named preset. 201 on first install, 200 on update or reinstall. 404 if not in the index. 409 if a built-in or custom product already uses that name. 422 on SHA-256 mismatch or validation failure. | +| `GET` | `/api/v1/presets` | List all presets with per-preset status. `?refresh=true` forces a re-fetch of the index. 502 if the index cannot be fetched. | +| `GET` | `/api/v1/presets/{name}` | Preview a preset — metadata, template, and status. 404 if not in the index. 502 if the index cannot be fetched. 422 if the definition cannot be fetched, verified or parsed. | +| `POST` | `/api/v1/presets/{name}/install` | Install or update the named preset. 201 on first install, 200 on update or reinstall. 404 if not in the index. 409 if a built-in or custom product already uses that name. 422 on SHA-256 mismatch or validation failure. 502 if the index cannot be fetched. | To uninstall, use `DELETE /api/v1/products/{name}`. @@ -172,23 +175,31 @@ The official library ships `Rendering`-category presets — `maya-layer-render`, [`sqi-submitter`](dcc-submitters.md) in-application submitters, plus `mistika-boutique-render`, `mistika-vr-render` and `mistika-workflows-render`, which have no in-application submitter and are submitted from the web UI or -the API. -Each declares its parameters using the [parameter convention -contract](dcc-submitters.md#the-parameter-convention-contract) (`SceneFile`, -`Frames`, `OutputDir`, plus per-host extras) so submitter pre-fill works out of -the box, and each gates on an operator-configured worker capability tag (see -[worker capability tags](worker-capabilities.md)). See -[`docs/dcc-submitters.md`](dcc-submitters.md) for the full reference, -including the chunking behavior and worker requirements per preset. +the API. Each names its parameters from the [parameter convention +contract](dcc-submitters.md#the-parameter-convention-contract) so submitter +pre-fill works out of the box — all nine declare `SceneFile` and `Frames`; an +output parameter is declared only where the command takes one (`OutputDir` on +the two Maya presets, `OutputPath` on Blender), alongside per-host extras such +as `Renderer`, `RenderLayer`, `RopPath` and `WriteNode`. Each gates on a worker +capability tag that `sqi-worker` auto-detects from a standard install with no +per-worker configuration — manual tags are needed only for nonstandard +install paths (see [capability +auto-detection](worker-capabilities.md#capability-auto-detection-built-in-dcc-detectors)). +See [`docs/dcc-submitters.md`](dcc-submitters.md) for the full reference, +including the chunking behavior and worker requirements per preset. Every +software tag a shipped preset requires must be emitted by a built-in detector — +enforced by `TestBuiltinDetectors_CoverPresets`, so a new reference preset +cannot ship without one. ## Transcoding reference presets The official library also ships five `Transcoding`-category presets under `presets/sqi/*.yaml` — plain ffmpeg jobs meant to be submitted directly rather than through a DCC submitter. Each gates on the `attr.worker.tag.ffmpeg = -"true"` capability tag (see [worker capability -tags](worker-capabilities.md)); the segmented variants add an OS gate on top -of that: +"true"` capability tag, which `sqi-worker` sets automatically on any worker +with `ffmpeg` on `PATH` (see [worker capability +tags](worker-capabilities.md)); the two shell-joined segmented variants add an +OS gate on top of that, and the portable one deliberately does not: - `ffmpeg-transcode` — converts one video file on one worker, start to finish. Base-spec OpenJD (declares no extensions), so it runs on any diff --git a/docs/products.md b/docs/products.md index efcb4c7c..6aab1c22 100644 --- a/docs/products.md +++ b/docs/products.md @@ -1,7 +1,8 @@ # Products -A **product** is a named, versioned wrapper around a verbatim OpenJD template. -It gives the template a stable identity (`name`), human-readable metadata +A **product** is a named, versioned wrapper around an OpenJD template, stored +unmodified apart from YAML re-serialization when it comes from a definition +file. It gives the template a stable identity (`name`), human-readable metadata (`title`, `description`, `category`, `version`), and a home in the catalog so clients can list and submit jobs without ever handling a raw template file. @@ -20,7 +21,7 @@ layer adds only the catalog envelope on top. │ name · title · description · category │ │ version · source │ │ ┌──────────────────────────────────────┐ │ - │ │ OpenJD template (verbatim) │ │ + │ │ OpenJD template │ │ │ │ parameters · steps · requirements │ │ │ └──────────────────────────────────────┘ │ └────────────────────────────────────────────┘ @@ -33,8 +34,10 @@ contains a **snapshot** of the template at submission time, so later edits to th product do not affect running or queued jobs. A product submission also accepts the same optional per-job overrides as a raw -`POST /api/v1/jobs`: `owner`, `submitter`, `priority`, `project`, and the retry -policy `max_attempts`, `retry_delay_seconds`, `failure_limit`. Each is optional; +`POST /api/v1/jobs`: `owner`, `submitter`, `priority`, `project`, `depends_on` +(IDs of upstream jobs in the same farm; the job starts blocked until they +complete), and the retry policy `max_attempts`, `retry_delay_seconds`, +`failure_limit`. Each is optional; an omitted field inherits the queue → farm → server default. See `internal/api/openapi.yaml` (`SubmitProductJobRequest`) for the authoritative wire contract. @@ -71,13 +74,17 @@ template: - name: Run script: embeddedFiles: - - name: script.py + # name is an OpenJD and is the key Task.File + # references resolve against, so it cannot contain a dot. The + # on-disk basename goes in filename. + - name: script type: TEXT + filename: script.py data: "{{Param.Script}}" actions: onRun: command: "{{Param.Interpreter}}" - args: ["script.py"] + args: ["{{Task.File.script}}"] ``` ### Metadata fields @@ -94,8 +101,17 @@ template: The `name` slug constrains to `^[a-z0-9][a-z0-9_-]*(/[a-z0-9][a-z0-9_-]*)?$`. The inline template is re-serialized and fully validated (via `openjd.Parse` + -`openjd.ValidateWithOptions`) when the definition is parsed — a malformed -template is rejected at load time. +`openjd.ValidateWithBudget`) when the definition is parsed — a malformed +template is rejected at load time. Validation is bounded: `ParseDefinition` and +`ValidateTemplate` take a **required** `product.ValidateOptions` carrying the +operator's configured EXPR limits (`openjd.expr_*`) and a per-request wall-clock +deadline (`openjd.expr_submission_deadline`). Every HTTP route that reaches this +package sets both. Two callers pass `EnforceLimits` alone, and neither is a +request path: the built-in loader (`internal/product/builtins.go`), which runs +from package init before any configuration exists, and `internal/presetgen`, +the offline index-build tool. A template that breaches +the deadline is a `503`, not a `400` — the same body would validate on an idle +server. ### Writing a `readme` @@ -168,7 +184,7 @@ Demonstrates the minimal product shape: one `STRING` parameter with a ### `python` — Run a Python Script Demonstrates two parameters (`Interpreter` and `Script`), an OpenJD -`embeddedFiles` block that materialises the script body as a file named +`embeddedFiles` block that materializes the script body as a file named `script.py`, and a configurable interpreter path defaulting to `python3`. ### `container` — Run a Docker Image @@ -186,7 +202,7 @@ Boutique/VR/Workflows — see way: a `hostRequirements.attributes` entry requiring `attr.worker.tag.` with `anyOf: ["true"]`. `sqi-worker` auto-detects a standard install of each of those applications and advertises the matching tag (e.g. `maya`) with -value `"true"` with no configuration — see [Capability +value `"true"` and no configuration — see [Capability auto-detection](worker-capabilities.md#capability-auto-detection-built-in-dcc-detectors) — which satisfies the `anyOf: ["true"]` match above directly, so a worker with a standard install matches these built-in gates with zero per-worker @@ -222,14 +238,17 @@ with `source: installed`. See [`docs/preset-library.md`](preset-library.md) for the full guide, including the index format, configuration, and the browse → preview → install flow. -The official library's `Rendering`-category DCC presets (`maya-layer-render`, -`maya-scene-render`, `houdini-rop-render`, `nuke-write-render`, -`nuke-script-render`, `blender-batch-render`) exist to give the -[`sqi-submitter`](dcc-submitters.md) in-application submitters something real to -target. They declare their -parameters (`SceneFile`, `Frames`, `OutputDir`, plus per-host extras like -`Renderer`/`RopPath`/`WriteNode`) following a documented, versioned, -additive-only naming convention — duplicate one and keep the parameter names +The official library ships nine `Rendering`-category presets. Six of them — +`maya-layer-render`, `maya-scene-render`, `houdini-rop-render`, +`nuke-write-render`, `nuke-script-render`, `blender-batch-render` — exist to give +the [`sqi-submitter`](dcc-submitters.md) in-application submitters something real +to target. The other three, `mistika-boutique-render`, `mistika-vr-render` and +`mistika-workflows-render`, have no in-application submitter and are submitted +from the web UI or the API. They name their parameters from a documented, +versioned, additive-only convention — `SceneFile` and `Frames` on all nine, an +output parameter only where the command takes one (`OutputDir` on the Maya +presets, `OutputPath` on Blender), plus per-host extras like +`Renderer`/`RopPath`/`WriteNode` — duplicate one and keep the parameter names to keep submitter pre-fill working. Full reference: [`docs/dcc-submitters.md`](dcc-submitters.md). @@ -240,9 +259,12 @@ The library also ships five `Transcoding`-category ffmpeg presets submitted from a host application and declare no scene-file parameters for submitter pre-fill to bind — install and submit them directly from the web UI or REST API. Each gates on the `attr.worker.tag.ffmpeg = "true"` -capability tag; the three segmented variants split a source into slices -across the farm and differ only in how they join the slices back together -(bash, PowerShell, or a shell-free EXPR template). Full reference: +capability tag, which `sqi-worker` sets automatically on any worker with +`ffmpeg` on `PATH` (see [worker capability +tags](worker-capabilities.md)); the three segmented variants split a source +into slices across the farm and differ in how they join the slices back +together (bash, PowerShell, or a shell-free EXPR template), which OS they can +run on, and whether they delete the slice files afterwards. Full reference: [`docs/preset-library.md`](preset-library.md#transcoding-reference-presets). Key points about installed products: @@ -262,7 +284,7 @@ Key points about installed products: `name` is the stable identity of a product across its lifetime. The `version` string (e.g. `1.0.0`) is stored alongside the template and is available for future tooling to detect when an installed product's template has been -superseded by a newer release. No automatic update behaviour is implemented in +superseded by a newer release. No automatic update behavior is implemented in Phase 2; `version` is a label only. --- @@ -298,7 +320,10 @@ Request body: `format` is `"yaml"` (default) or `"json"`. -Responses: `201 Created` (product), `400 Bad Request`, `409 Conflict`. +Responses: `201 Created` (product), `400 Bad Request`, `403 Forbidden` (auth on, +missing `products.manage`), `409 Conflict`, `503 Service Unavailable` (template +validation exceeded `openjd.expr_submission_deadline` — retry; the same body may +validate on an idle server). ### `GET /api/v1/products/{name}` @@ -315,7 +340,7 @@ Replaces the mutable fields of a stored product. Built-ins return Request body: same shape as `POST /api/v1/products` (name in path takes precedence over name in body). -Responses: `200 OK`, `400`, `403`, `404`. +Responses: `200 OK`, `400`, `403`, `404`, `503` (validation deadline exceeded). ### `DELETE /api/v1/products/{name}` @@ -351,14 +376,22 @@ Response: `200 OK`, array of `ProductParameter` objects in template order: "label": "Interpreter", "group_label": "", "decimals": null - } + }, + "file_filters": null, + "file_filter_default": null, + "item": null } ] ``` -Responses: `200 OK` (array), `404 Not Found` (product not found), -`422 Unprocessable Entity` (product's stored template cannot be parsed — the -template is stored verbatim and its validity is only checked on this call). +`file_filters` / `file_filter_default` carry a PATH parameter's file-dialog +filters (the Mistika presets use them to offer `*.rnd`); `item` carries a +`LIST[*]` parameter's per-element constraints and is `null` for a scalar. + +Responses: `200 OK` (array), `403 Forbidden`, `404 Not Found` (product not +found), `422 Unprocessable Entity` (the product's stored template cannot be +parsed; templates are validated at create/update time, so this normally only +appears for a row written by an older or external path). ### `POST /api/v1/products/{name}/jobs` @@ -389,10 +422,15 @@ Request body: overrides the job name from the product's template; when omitted, the template's own name is used. The web submission form defaults it to `" "`. `parameters` is a flat `string→string` map; -keys must match the parameter names declared in the product's template. +keys must match the parameter names declared in the product's template. A +`LIST[*]` parameter's value is the JSON encoding of the list, still as a string +— for example `"[\"main\",\"closeup\"]"`. Missing keys with defaults are filled +automatically; a missing required key returns 422. Responses: `201 Created` (`Job` object, same shape as `POST /api/v1/jobs`), -`400`, `404`, `422 Unprocessable Entity` (template/parameter validation failure). +`400`, `403`, `404`, `422 Unprocessable Entity` (template/parameter validation +failure), `503` (expression evaluation exceeded +`openjd.expr_submission_deadline`). --- diff --git a/docs/python-client.md b/docs/python-client.md index cb298be5..7b485b48 100644 --- a/docs/python-client.md +++ b/docs/python-client.md @@ -32,6 +32,7 @@ contract is the OpenAPI spec described in [`api.md`](./api.md). - [Tasks and logs](#tasks-and-logs) - [Workers](#workers) - [Farm, queue, and resource CRUD](#farm-queue-and-resource-crud) +- [Products](#products) - [Live events (WebSocket, `ws` extra)](#live-events-websocket-ws-extra) - [Conveniences](#conveniences) @@ -102,6 +103,7 @@ SqiError ├── SqiTimeoutError # request timed out / wait_for_job deadline └── APIError # any non-2xx response ├── BadRequestError # 400 + ├── SqiAuthError # 401 / 403 (missing or rejected credential) ├── NotFoundError # 404 ├── ConflictError # 409 ├── ValidationError # 422 @@ -110,7 +112,7 @@ SqiError ``` `APIError` carries `status`, `title`, `detail`, and `request_id` (parsed from the -RFC 7807 `application/problem+json` body or the `X-Request-Id` header). `str(exc)` +RFC 7807 `application/problem+json` body or the `X-Request-ID` header). `str(exc)` includes the status, detail, and request ID so failures are diagnosable in logs. ```python @@ -148,9 +150,10 @@ Status filters accept either the enum or its wire string: `sqi.list_jobs(status=JobStatus.RUNNING)` and `sqi.list_jobs(status="running")` are equivalent. `None` filters are omitted from the request entirely. -> **Note:** farms, storage locations, and usage pools are returned by the -> server as bare arrays (no pagination), so their `list_*` methods return a plain -> `list[T]`. Only queues, jobs, tasks, and workers are paginated. +> **Note:** farms, storage locations, compute locations, usage pools, and +> products are returned by the server as bare arrays (no pagination), so their +> `list_*` methods return a plain `list[T]`. Only queues, jobs, tasks, and +> workers are paginated. ## Submitting jobs @@ -264,6 +267,7 @@ for chunk in sqi.tail_task_logs(task_id, follow=True): | `get_worker(worker_id) -> Worker` | Worker detail, including `current_tasks`. | | `disable_worker(worker_id) -> WorkerAction \| None` | Drain and stop new assignments. | | `enable_worker(worker_id) -> WorkerAction \| None` | Re-enable a disabled worker. | +| `remove_worker(worker_id) -> None` | Hard-delete a worker record; `204` → `None`. Only offline workers, or disabled workers whose last heartbeat is older than the heartbeat-timeout window, are removable — an online or live-disabled worker raises `ConflictError`. | ```python for worker in sqi.iter_workers(status="online"): @@ -287,6 +291,7 @@ reset, so pass every field you want to keep. | Farms | `create_farm(*, name, description=None, max_concurrent_tasks=0, max_attempts=None, retry_delay_seconds=None, failure_limit=None)` | `list_farms() -> list[Farm]`, `iter_farms()` | `get_farm`, `update_farm`, `delete_farm` | | Queues | `create_queue(*, farm_id, name, description=None, priority=0, max_concurrent_tasks=0, paused=False, max_attempts=None, retry_delay_seconds=None, failure_limit=None)` | `list_queues(*, farm_id, paused, sort_by, sort_dir, limit, offset) -> Page[Queue]`, `iter_queues(...)` | `get_queue`, `update_queue`, `delete_queue` | | Storage locations | `create_storage_location(*, name, description=None, roots=None)` | `list_storage_locations() -> list[StorageLocation]`, `iter_storage_locations()` | `get_storage_location`, `update_storage_location`, `delete_storage_location` | +| Compute locations | `create_compute_location(*, name, description=None)` | `list_compute_locations() -> list[ComputeLocation]`, `iter_compute_locations()` | `get_compute_location`, `update_compute_location`, `delete_compute_location` | | Usage pools | `create_usage_pool(*, name, max_concurrent, server_hint=None)` | `list_usage_pools() -> list[UsagePool]`, `iter_usage_pools()` | `get_usage_pool`, `update_usage_pool`, `delete_usage_pool` | ```python @@ -303,6 +308,40 @@ client-side (raising `ValueError`) before sending. Every `UsagePool` response also carries read-only, server-computed `in_use` (active claims) and `available` (`max(max_concurrent - in_use, 0)`) fields for live utilization. +## Products + +Products are named, versioned wrappers around OpenJD templates stored in the +server's catalog. Eight methods cover them: + +| Method | Description | +|---|---| +| `list_products() -> list[Product]` | Every product (built-ins + custom); bare array, no pagination. | +| `iter_products() -> Iterator[Product]` | Iterator companion. | +| `get_product(name) -> Product` | One product by name; 404 → `NotFoundError`. | +| `create_product(*, name, template, format, title=None, description=None, category=None, version=None) -> Product` | Create a custom product from a raw OpenJD template. | +| `update_product(name, *, template, format, title=None, description=None, category=None, version=None) -> Product` | Full PUT replacement of a custom product. | +| `delete_product(name) -> None` | Delete a custom product; a built-in raises `SqiAuthError` (403). | +| `get_product_parameters(name) -> list[ProductParameter]` | The parsed job parameters — type, default, allowed values, and `user_interface` hints. 404 → `NotFoundError`; an unparseable stored template → `ValidationError` (422). | +| `submit_product_job(name, *, farm_id, queue_id, job_name=None, owner=None, submitter=None, priority=None, project=None, parameters=None, max_attempts=None, retry_delay_seconds=None, failure_limit=None, depends_on=None) -> Job` | Submit a job from a product. | + +```python +products = sqi.list_products() +params = sqi.get_product_parameters("python") +for p in params: + print(p.name, p.type, p.default) + +job = sqi.submit_product_job( + "python", + farm_id=farm_id, queue_id=queue_id, + job_name="My Script Run", + parameters={"Script": "print('hello')", "Interpreter": "python3"}, +) +``` + +`submit_product_job` uses the keyword `job_name=` (not `name=`) so it does not +shadow the positional product `name`; the wire field sent to the server is +`"name"`. + ## Live events (WebSocket, `ws` extra) With `pip install 'sqi-sdk[ws]'`, `SqiClient.events()` opens a live event @@ -319,9 +358,10 @@ with sqi.events() as stream: print(event.subject, event.seq, event.payload) ``` -Subjects: `jobs`, `jobs/{job-id}/tasks`, `tasks/{task-id}/logs`, `workers` -(see [`api.md`](./api.md#available-subjects) for payload shapes). A failed -subscription or server `error` frame is raised as `SqiError`. +Subjects: `jobs`, `jobs/{job-id}/tasks`, `tasks/{task-id}/logs`, `workers`, and +`diagnostics` (see [`api.md`](./api.md#available-subjects) for payload shapes). +Subscribing to `diagnostics` requires the `diagnostics.read` permission. A +failed subscription or server `error` frame is raised as `SqiError`. `tail_task_logs_live(task_id, from_seq=0) -> Iterator[LogChunk]` is the WebSocket-backed counterpart to `tail_task_logs`: @@ -331,8 +371,10 @@ for chunk in sqi.tail_task_logs_live(task_id): print(chunk.data, end="") ``` -Calling `events()`/`tail_task_logs_live` without the `ws` extra installed raises -`ImportError` naming the exact remedy (`pip install 'sqi-sdk[ws]'`). The live +Without the `ws` extra installed, `events()` itself succeeds — the `websockets` +import is deferred to `connect()`, so the `ImportError` naming the exact remedy +(`pip install 'sqi-sdk[ws]'`) is raised when the stream is entered as a context +manager, and on the first iteration of `tail_task_logs_live`. The live `tasks/{id}/logs` payload omits `id`/`nats_seq`/`received_at`, so those are zero-valued on the yielded `LogChunk`; `seq_num` and the content fields are set. @@ -358,3 +400,13 @@ job = sqi.submit_and_wait( if job.status != JobStatus.COMPLETED: # compare by value, not identity raise SystemExit(f"job {job.id} ended as {job.status}") ``` + +`me() -> Principal` returns the authenticated principal — `subject`, +`display_name`, `roles`, `permissions`, `kind`, and (for local accounts) +`username` — from `GET /api/v1/auth/me`. Gate on `permissions`, not `roles`: +check for `"jobs.submit_as"` before setting a job `owner` other than your own +user; without it the server responds `403`. With auth disabled the anonymous +superuser principal is returned. + +`server_version() -> ServerVersion` returns the server's build metadata from +`GET /api/v1/version`. diff --git a/docs/quickstart.md b/docs/quickstart.md index 04fef776..6023f1e9 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -79,7 +79,7 @@ Rather than the web UI, artists can submit directly from the DCC they're already working in. The [`sqi-submitter`](dcc-submitters.md) Python package adds an in-application submit dialog (native panel for Blender) to Maya, Houdini, Nuke, and Blender, driven by the same product catalog — including -six ready-to-install reference render presets — with scene path, frame +nine ready-to-install reference render presets — with scene path, frame range, and render-target pre-fill. See [DCC submitters](dcc-submitters.md) for installation per host. diff --git a/docs/release-runbook.md b/docs/release-runbook.md index 9b92240c..4d596036 100644 --- a/docs/release-runbook.md +++ b/docs/release-runbook.md @@ -4,8 +4,9 @@ This runbook captures the complete maintainer procedure for cutting a versioned `sqi` release. Follow the steps in order. Each external action has a **Verify:** line — do not proceed past a step until verification passes. -This document covers Steps 2–9 of the release plan. Step 1 (writing this file) -is already done. Step 10 (committing the runbook) closes the loop. +The step numbering below is historical (it follows the original release-plan +task order) and starts at Step 2. Steps 2, 3, 3b, 6b and 7 are one-time setup; +Steps 4-9 are the per-release procedure. --- @@ -179,9 +180,10 @@ GOVERSION=$(go version | awk '{print $3}') \ ``` **Expected:** the snapshot build completes successfully. Both `sqi-server` and -`sqi-worker` are produced for all target platforms (`linux/amd64`, `linux/arm64`, -`darwin/amd64`, `darwin/arm64`, `windows/amd64`). All four Docker image variants -build (`-amd64`/`-arm64` per image, plus the manifest tags). +`sqi-worker` are produced for all six target platforms (`linux/amd64`, +`linux/arm64`, `darwin/amd64`, `darwin/arm64`, `windows/amd64`, +`windows/arm64`). All four Docker image variants build (`-amd64`/`-arm64` per +image, plus the manifest tags). --- @@ -339,11 +341,11 @@ Also verify `sqi-submitter` (once its version has been bumped past the `.dev0` placeholder per Step 6): ```bash -python -m pip install --no-cache-dir "sqi-submitter==" +python -m pip install --no-cache-dir "sqi-submitter==$VERSION" python -c "import sqi_submitter; print(sqi_submitter.__version__)" ``` -Expected output: `` matching the tag. +Expected output: `$VERSION`. --- diff --git a/docs/roadmap.md b/docs/roadmap.md index cdf54479..6218a026 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -88,7 +88,7 @@ Configuration cascades: farm defaults → queue overrides, with retry policy (ma reservation; omitting it reserves the whole machine (one task per worker). The server tracks committed cores in the database; the ledger rebuilds instantly on restart. - - *Deferred:* worker drain/headroom signal, head-of-line reservation for + - *Deferred:* worker drain/headroom signal, head-of-line reservation for large tasks, `amount.worker.vcpu.max`, memory/GPU dimensions. --- @@ -98,7 +98,7 @@ Configuration cascades: farm defaults → queue overrides, with retry policy (ma `sqi` adopts the [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications) (OpenJD) format as its native job execution format. **Benefits:** -- Studios authoring jobs for other OpenJD-compatible systems can submit to `sqi` unchanged +- Studios authoring jobs for other OpenJD-compatible systems can submit to `sqi` unchanged, provided the template does not opt into an extension `sqi` has not implemented — those are rejected by design rather than accepted and misinterpreted (the official `EXPR` expression-language extension **is** implemented and supported) — `sqi` accepts every valid base-spec template in the official conformance suite, though it is still more permissive than the spec about rejecting *invalid* ones (tracked in `test/conformance/baseline.txt`, measured in [`docs/openjd-conformance.md`](openjd-conformance.md)) - Standardized path mapping, parameter spaces, and execution semantics - Clear separation between job description and job authoring (the product system) @@ -161,11 +161,31 @@ S3-compatible store reachable by the operator's chosen sync tool. ### Path Translation Modes -- **OpenJD** (preferred): Standard path mapping file written into each Session. Applications that support OpenJD natively consume it directly. -- **Resolved**: All paths resolved to concrete paths before command construction. Universal for applications with no path mapping support. -- **Command arg**: Path pairs passed as explicit arguments (e.g., Maya workspace remapping). -- **Environment**: Path mappings via environment variables. -- **Staged**: Pre-job staging to worker-local storage for cloud workers without direct access to source storage. +Path translation rides the `SQI_PATH_TRANSLATION` extension and offers five +delivery mechanisms (deliveries execute in fixed order and are mutually +compatible — a product can declare all five): + +- **`translation_file`** (preferred): Native OpenJD `pathmapping-1.0` file + written into each Session, served via `{{Session.PathMappingRulesFile}}`. + Applications that support OpenJD path mapping natively consume it directly. +- **`swap_in_place`**: String substitution of path parameters in the template. + Universal for applications with no path-mapping support. sqi convenience, + not in the OpenJD spec. +- **`command_flags`**: Individual `src`/`dest` pairs appended as command-line + flags (e.g., Maya workspace remapping). +- **`environment`**: Path mappings delivered via an environment variable. +- **`stage_locally`**: Job-level PATH parameters staged to worker-local + scratch before the run and copied back after, for cloud workers without + direct access to source storage. Works with no worker configuration — an + unconfigured worker falls back to a TEMP scratch directory and sqi's own + built-in copy — but a farm spanning multiple compute locations needs an + explicit `staging.scratch_dir` and `staging.sync_command` + (`rsync`/`aws-cli`/etc.) for real remote transfer. + +`swap_in_place` and `translation_file` are the default when no +`SQI_PATH_TRANSLATION` extension is declared. Full reference: +[`products.md`](products.md#path-translation) and +[`openjd-extensions/path-translation.md`](openjd-extensions/path-translation.md). ### What `sqi` does not do @@ -221,7 +241,7 @@ NATS can run embedded within `sqi-server` (simple mode) or as a separate cluster - Product/preset definition system (YAML/JSON) — a thin catalog over OpenJD templates, with embedded Script/Python/Container built-ins - Preset library integration — static JSON index at a configurable URL (default: official community library on GitHub Pages); browse presets in the Admin hub with per-preset status (not installed / installed / update available); preview the definition and install as a product (`source: installed`) in one click; SHA-256 integrity and update-detection check on install; read-only installed products, uninstallable, with Duplicate-to-custom available on every product - Web UI product management editor and a product-driven submission form (parameter form generated from the selected product) -- Additional path translation modes (resolved, command-arg, environment, staged) as the `SQI_PATH_TRANSLATION` vendor extension +- Path translation deliveries (`swap_in_place`, `translation_file`, `command_flags`, `environment`, `stage_locally`) as the `SQI_PATH_TRANSLATION` vendor extension - S3-compatible storage support (thin layer: derived type, root validation, path staging via operator sync tool) - DCC submitter framework — in-application submitters for Maya, Houdini, Nuke, and Blender (the `sqi-submitter` Python package), built on the Python client - Compute location registry and step-level affinity (native OpenJD `attr.worker.computelocation`) @@ -251,6 +271,23 @@ before Phase 3. See [docs/auth.md](auth.md) for the model and setup. [docs/worker-configuration.md](worker-configuration.md) for the model, setup, and known gaps. +### Post-Phase-3, pre-v0.3: OpenJD `EXPR` and expanded reference presets — complete, unreleased + +- **OpenJD `EXPR` extension** — the official expression-language extension is + fully implemented and `StatusSupported`: expression core, type system, + collections, comprehensions, the ~100-function standard library, path + mapping, template integration (scopes, `let` bindings, bounded evaluation + with operator-configurable limits), RFC 0007 extended parameter types, and + the web `*_LIST` widgets. EXPR templates are accepted, submitted, dispatched + and executed, with expressions resolved on the worker at phase 3. See + [`docs/openjd-extensions/expr.md`](openjd-extensions/expr.md). +- **ffmpeg reference presets** — transcode, sequence-encode, and + segment-transcode (bash/PowerShell/EXPR) added to `presets/sqi/`; the + segment-transcode EXPR variant is the first shipped preset to declare + `extensions: [EXPR]`. +- **Mistika reference presets** — Boutique, VR, and Workflows render presets + added to `presets/sqi/`, each using the `SQI_CHUNK_BOUNDS` extension. + ### Phase 4: Production Hardening (v0.4 — beta) - PostgreSQL state store option diff --git a/docs/storage-s3.md b/docs/storage-s3.md index cc2de3d0..3fe3ed5c 100644 --- a/docs/storage-s3.md +++ b/docs/storage-s3.md @@ -266,7 +266,7 @@ use `stage_locally`. The workflow is: 4. Configure each cloud worker with `staging.scratch_dir` and `staging.sync_command`. -Outputs are copied back after each task automatically by the same sync command +Outputs are automatically copied back after each task by the same sync command (direction reversed: `dest` ← `src`). See [`docs/openjd-extensions/path-translation.md`](openjd-extensions/path-translation.md) diff --git a/docs/web-accessibility.md b/docs/web-accessibility.md index 4c908441..b78e0918 100644 --- a/docs/web-accessibility.md +++ b/docs/web-accessibility.md @@ -7,8 +7,10 @@ further is welcome. The audience is operators using a keyboard-and-screen-reader workflow or working in varied lighting on varied displays, not a formal WCAG 2.1 AA certification -effort. The UI has no auth flows or modal-heavy interactions today, so the -baseline is correspondingly focused. +effort. The UI now has a login flow (rendered in place of the app shell when the +server has `auth.enabled: true` and no session is present) and a small number of +confirmation dialogs, both of which are in scope for the keyboard and +text-alternative commitments below. --- diff --git a/docs/web-development.md b/docs/web-development.md index 4511c77e..a1d323e4 100644 --- a/docs/web-development.md +++ b/docs/web-development.md @@ -25,7 +25,7 @@ field of `web/package.json`. From the repo root: ```sh nvm use # selects the Node version from .nvmrc cd web -npm install # install dependencies (run once, and after dependency changes) +npm ci # install dependencies exactly as locked (run once, and after dependency changes) ``` --- @@ -44,8 +44,10 @@ repo root unless noted. make run # or, after `make build`: ./bin/sqi-server serve -# Terminal 2 — a worker, so submitted jobs actually run -./bin/sqi-worker --server http://localhost:8080 +# Terminal 2 — a worker, so submitted jobs actually run. +# On a LAN it finds the server via mDNS; point it explicitly with the NATS URL +# if discovery is off or the server is elsewhere. +SQI_WORKER_NATS_URL=nats://127.0.0.1:4222 ./bin/sqi-worker start # (build it first with `make build` if needed) # Terminal 3 — the Vite dev server (from web/) diff --git a/docs/worker-capabilities.md b/docs/worker-capabilities.md index 5bdb38c3..f0f2ab37 100644 --- a/docs/worker-capabilities.md +++ b/docs/worker-capabilities.md @@ -113,15 +113,14 @@ GPU-capable on macOS or Windows. In addition to the hardware/OS tags above, `sqi-worker` runs a second, declarative detection engine (`internal/worker/capabilities`) that looks for installed creative applications — Maya, Nuke, Houdini, Blender, Mistika -(Boutique/Ultima, VR, Workflows) and ffmpeg — and -advertises a tag with value `"true"` automatically, with no per-worker -configuration. This makes the software actually installed on a worker visible -without hand-editing its config, and it's enough on its own to satisfy the -`anyOf: ["true"]` gate the shipped reference presets declare — a standard -install matches those presets with zero configuration. Run -`sqi-worker capabilities` any time to see what was found. See -[`docs/dcc-submitters.md`](dcc-submitters.md#reference-presets) for how these -auto-detected tags relate to the reference presets. +(Boutique/Ultima, VR, Workflows) and ffmpeg — and advertises a tag with value +`"true"` automatically, with no per-worker configuration. This makes the +software actually installed on a worker visible without hand-editing its +config, and it's enough on its own to satisfy the `anyOf: ["true"]` gate the +shipped reference presets declare — a standard install matches those presets +with zero configuration. Run `sqi-worker capabilities` any time to see what +was found. See [`docs/dcc-submitters.md`](dcc-submitters.md#reference-presets) +for how these auto-detected tags relate to the reference presets. ### How it runs @@ -179,8 +178,7 @@ a path-based check also matches on that host. Every software tag a shipped reference preset requires (`presets/sqi/*.yaml`'s `attr.worker.tag.`) must be emitted by some -built-in detector — enforced by -`TestBuiltinDetectors_CoverPresets` in +built-in detector — enforced by `TestBuiltinDetectors_CoverPresets` in [`internal/worker/capabilities/builtins_test.go`](https://github.com/uberware/sqi/blob/main/internal/worker/capabilities/builtins_test.go), so a new default preset cannot ship without a matching detector (see [Adding a new auto-detected tag](development.md#adding-a-new-capability-tag-to-auto-detection) @@ -239,13 +237,13 @@ os_version 22.04 auto The `VALUE` column is the tag's advertised value (bare tags resolve to `true`; hardware tags like `os`/`os_version` carry their detected string). The `SOURCE` -column is one of `auto` (the hardware probe's own `Tags` entries -— just `os` and, when detected, `os_version`), `builtin:`, `custom`, or -`manual`. This is the first thing to run -when a worker isn't picking up jobs you'd expect it to — it answers "why -isn't my worker getting Maya jobs?" without needing to connect to a server or -inspect logs. `sqi-worker start --dry-run` includes the same detected -capabilities as part of its larger config-and-capabilities summary. +column is one of `auto` (the hardware probe's own `Tags` entries — just `os` +and, when detected, `os_version`), `builtin:`, `custom`, or `manual`. +This is the first thing to run when a worker isn't picking up jobs you'd +expect it to — it answers "why isn't my worker getting Maya jobs?" without +needing to connect to a server or inspect logs. `sqi-worker start --dry-run` +includes the same detected capabilities as part of its larger +config-and-capabilities summary. --- diff --git a/docs/worker-configuration.md b/docs/worker-configuration.md index 39ed4ebe..2fdf9a25 100644 --- a/docs/worker-configuration.md +++ b/docs/worker-configuration.md @@ -12,7 +12,9 @@ layers overriding earlier ones: `SQI_DIAGNOSTICS_ENABLED` and `staging.defaults` uses `SQI_STAGING_DEFAULTS`, both with no `WORKER` infix — see the `diagnostics` and `staging` sections.) -4. **CLI flags** — highest priority; available on the `start` subcommand. +4. **CLI flags** — highest priority. `--config`/`-c`, `--log-level` and + `--log-format` are root flags available on every subcommand; `--dry-run` and + `--nats-insecure-skip-verify` belong to `start` only. Print the effective merged configuration at any time with: @@ -209,7 +211,8 @@ worker: | **Default** | `~/.sqi/worker` (Linux/macOS); `%USERPROFILE%\.sqi\worker` (Windows) | | **Env var** | `SQI_WORKER_DATA_DIR` | -Directory used to persist the worker ID file (`worker.id`) ONLY. Created +Directory used to persist the worker ID file (`worker.id`), and on Windows the +DPAPI-encrypted run-as-user credential store (`\isolation\`). Created automatically on first start, and never widened for run-as-user traversal — it stays private (0700) for as long as the worker exists. @@ -221,9 +224,12 @@ Each worker instance needs its own `data_dir`: two workers sharing one would load the same `worker.id` and collide on the server. This is the key setting when [running multiple workers on one host](#running-multiple-workers-on-one-host). -Session working directories are a SEPARATE location — see -[`worker.session_dir`](#workersession_dir) below — not a child of `data_dir` -as they were before run-as-user isolation existed. +Session working directories have their own setting — see +[`worker.session_dir`](#workersession_dir) below. They are moved out from under +`data_dir` for any worker that could actually use run-as-user isolation (a root +POSIX worker, or any Windows worker); a non-root POSIX worker with +`session_dir` unset still keeps them at `/sessions`, the pre-split +location, because isolation cannot function there anyway. ```yaml worker: @@ -249,7 +255,7 @@ traversable by whichever run-as-user identity a session resolves to, while Left unset, the effective value is resolved at startup: -- **Running as root** — `/var/lib/sqi-worker-sessions`, created traversable +- **Running as root (POSIX)** — `/var/lib/sqi-worker-sessions`, created traversable (`0711`) from birth. Deliberately a SIBLING of, never a descendant of, `data_dir`'s own HOME-unset fallback (`/var/lib/sqi-worker`): nesting the two would make `LoadOrCreateWorkerID`'s own `0700` `data_dir` an ancestor @@ -257,11 +263,17 @@ Left unset, the effective value is resolved at startup: refuse to start over a directory sqi itself just created. Its ancestors (`/var`, `/var/lib`) are `0755` on every real Linux/macOS installation, so nothing needs to be created or widened specifically for this. -- **Otherwise** — `/sessions`, created at `0750` (the location and - mode used before this split existed). Real run-as-user isolation cannot - function without root regardless of directory permissions, so there is - nothing to protect by moving it, or widening it, for a worker that can - never use it anyway. +- **Windows (any account)** — `%ProgramData%\sqi\worker\sessions`, chosen + regardless of privilege: a worker running as LocalSystem resolves its data + directory under `System32\config\systemprofile`, which is the wrong place for + render scratch. Directory modes are inert on Windows; a session directory's + real protection is the protected NTFS DACL applied beneath this root (see + [Windows](#windows) below). +- **Otherwise (non-root POSIX)** — `/sessions`, created at `0750` + (the location and mode used before this split existed). Real run-as-user + isolation cannot function without root regardless of directory permissions, + so there is nothing to protect by moving it, or widening it, for a worker + that can never use it anyway. ```yaml worker: @@ -468,9 +480,12 @@ worker: Restrict this worker to serving specific queue IDs. The worker keeps one outstanding lease request per listed queue (`work.lease.`). When -empty (the default), the worker issues a single lease request using an empty -queue ID, which the server treats as a wildcard. Set this on heterogeneous -farms where some workers specialise in a subset of queues. +empty (the default), the worker issues a single lease request on the reserved +subject `work.lease._any` — an empty leaf would produce the invalid subject +`work.lease.` with no responders. The server selects tasks farm-wide for that +token and gates by worker eligibility, so a queue-unaffiliated worker is +matched to any queue's ready work. Set this on heterogeneous farms where some +workers specialise in a subset of queues. ```yaml worker: @@ -574,7 +589,7 @@ worker: |---|---| | **Type** | `string` | | **Default** | `"logon_user"` | -| **Accepted values** | `logon_user`, `s4u` | +| **Accepted values** | `logon_user` (`s4u` is recognised and refused; any other value fails Windows provider construction) | | **Env var** | `SQI_WORKER_ISOLATION_PROVIDER` | Selects the Windows credential mechanism. **Ignored on POSIX** — setting it @@ -838,6 +853,17 @@ budget rather than equal to them. Matching the server is the floor, not a guarantee: an accepted job can still exhaust a worker that passes every comparison, and no configuration on either side makes that impossible. +**Two of the five defaults have zero headroom against the server's, so raise +the workers first.** `expr.assignment_positions` (10,000) is exactly the +server's `openjd.expr_template_positions` default, and `expr.let_retained_bytes` +(10,000,000) is exactly its `openjd.expr_template_retained_bytes` default. +Raising either of those two server keys by any amount therefore withholds EXPR +work from *every* worker still on the shipped defaults, immediately and +farm-wide. The other three ship with real headroom (100x the server's operation +budget, 20x its memory budget, 2x its template-retained-bytes budget for +`expr.assignment_retained_bytes`). Roll the worker value out first, confirm the +registration `WARN` is gone, then raise the server. + **2. `operation_limit` and `assignment_positions` multiply.** The cumulative operation ceiling for one assignment is their product — 10¹⁰ at the defaults (1,000,000 x 10,000), and 10¹² if both are raised to their maxima @@ -1004,10 +1030,11 @@ expr: ## `capabilities` — Software auto-detection -Configures the built-in DCC detectors (Maya, Nuke, Houdini, Blender) that run -automatically at startup and advertise a `key=true` tag with no per-worker -configuration, plus any custom detectors for in-house tools. Full reference, -including the detector schema and the tag/version model: +Configures the eight built-in software detectors (Maya, Nuke, Houdini, +Blender, Mistika Boutique/Ultima, Mistika VR, Mistika Workflows, and ffmpeg) +that run automatically at startup and advertise a `key=true` tag with no +per-worker configuration, plus any custom detectors for in-house tools. Full +reference, including the detector schema and the tag/version model: [`docs/worker-capabilities.md`](worker-capabilities.md#capability-auto-detection-built-in-dcc-detectors). ### `capabilities.detect` @@ -1045,9 +1072,10 @@ capabilities: | **Env var** | `SQI_WORKER_CAPABILITIES_DISABLE` (comma-separated, appended to any config-file entries) | Built-in tag names to turn off, by exact tag (`maya`, `nuke`, `houdini`, -`blender`). Use this when a built-in misfires on a nonstandard host layout — -typically paired with a `capabilities.detect` entry for the same tag that -supplies more specific checks. +`blender`, `ffmpeg`, `mistika`, `mistikavr`, `mistikaworkflows`). Use this +when a built-in misfires on a nonstandard host layout — typically paired with +a `capabilities.detect` entry for the same tag that supplies more specific +checks. ```yaml capabilities: diff --git a/docs/worker-docker.md b/docs/worker-docker.md index e44b5b1d..fe56ee5c 100644 --- a/docs/worker-docker.md +++ b/docs/worker-docker.md @@ -335,6 +335,7 @@ docker build \ --build-arg VERSION=dev \ --build-arg COMMIT=$(git rev-parse --short HEAD) \ --build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) \ + --build-arg GOVERSION=$(go version | awk '{print $3}') \ -t sqi-worker:dev \ . ``` diff --git a/internal/README.md b/internal/README.md index e56de8ff..8f751cca 100644 --- a/internal/README.md +++ b/internal/README.md @@ -2,20 +2,50 @@ Server-internal Go packages. Go's `internal` rule makes these importable only by code in this module — they are not part of any public API and may change without notice. Anything that needs to be consumed from outside the module belongs in `pkg/` instead. +## Dependency direction + +`cmd/` → `internal/server` → (`internal/api`, `internal/scheduler`) → +(`internal/store`, `internal/bus`). `internal/server` is the only wiring layer: +it constructs the store, bus, scheduler, hub, auth chain and router and hands +them to each other. Leaf packages (`internal/store`, `internal/bus`, +`internal/log`, `internal/health`, `internal/metrics`, `internal/version`) +import no other `internal` package. Two near-leaves are worth naming: +`internal/diag` imports only `internal/log`, and `internal/config` imports +`internal/auth/oidc`, `internal/auth/policy`, and `internal/auth/rolemap` to +validate auth settings. + +**Concrete types never cross a package boundary.** Handlers and the scheduler +receive the `store.Store` *interface*, never `*sqlite.Store`, so tests inject +`internal/store/fake`. `internal/store/sqlite` is imported only by +`internal/server` and `cmd/`. + +**`internal/openjd` imports `internal/store`, so `internal/store` can never +import `internal/openjd`.** That is why the task state machine lives in +`internal/store/statemachine.go` while the step state machine lives in +`internal/openjd/statemachine.go`. It is also why the worker binary — which must +not link `internal/store` — cannot import `internal/openjd`, and why the shared +expression leaves `internal/openjd/expr` and `internal/openjd/intrange` exist. + ## Packages | Package | Purpose | |---|---| | `internal/api` | REST surface: chi router, handlers, request/response wire types, error shape | -| `internal/bus` | Embedded NATS JetStream broker and the typed client wrapper over it | -| `internal/config` | Layered runtime configuration (defaults, file, env) and validation | +| `internal/auth` | Opt-in authentication and authorization: local passwords, sessions, API keys, RBAC policy, role mapping, LDAP/AD and OIDC/SSO | +| `internal/bus` | Embedded NATS broker (JetStream streams for task status/logs/cancel and worker registration/heartbeat/deregister; plain core NATS for `work.lease.` request/reply and `worker.diag.`) and the typed client wrapper over it | +| `internal/config` | Layered runtime configuration (defaults → file → env → flags) and validation | +| `internal/diag` | Bounded in-memory ring buffer of diagnostic (operational) log records from the server and connected workers | | `internal/discovery` | mDNS responder that advertises the running server on the local network | -| `internal/health` | Liveness (`/healthz`) and readiness (`/readyz`) checks | +| `internal/fsutil` | Filesystem helpers shared across packages (AppleDouble sidecar filtering) | +| `internal/health` | Liveness (`/healthz`, no checkers, always 200) and readiness (`/readyz`, runs the registered `sqlite` and `nats` checkers concurrently under a 5 s deadline; 503 `degraded` on any failure) | | `internal/log` | `slog`-based structured logging setup and helpers | | `internal/metrics` | Prometheus metric definitions and registration | | `internal/middleware` | `net/http` middleware (request logging, recovery, etc.) | | `internal/openjd` | OpenJD template parsing, validation, parameter-space expansion, path mapping | -| `internal/scheduler` | Assignment loop, worker registry, and scheduling policy | +| `internal/presetgen` | Generates the shipped preset catalog artifacts from the reference preset definitions | +| `internal/presetlib` | Fetches and caches the remote preset index; verifies SHA-256 on install | +| `internal/product` | Product catalog: embedded built-ins overlaid on stored custom/installed products | +| `internal/scheduler` | Lease handler and scheduling policy, worker registry, heartbeat/retention/unschedulable sweeps, cross-job dependency reconcile, and the server-side NATS consumers for task status, task logs, and worker diagnostics | | `internal/server` | Component lifecycle: starting and stopping the server's subsystems | | `internal/store` | Storage interface, SQLite-backed implementation, migrations, domain types, fake | | `internal/ui` | Serves the embedded web UI bundle with single-page-application fallback routing | @@ -28,8 +58,13 @@ Server-internal Go packages. Go's `internal` rule makes these importable only by The worker agent is decomposed into focused subpackages: `capabilities` and `registration` (self-reporting and registering with the server), `discovery` (locating the server via mDNS), `natsclient` and `protocol` (NATS transport and -message types), `pull` (the assignment pull loop), `heartbeat`, `executor` and -`session` (OpenJD session lifecycle and bare-metal process execution), `pathmap` -(storage-location path resolution), `logstreamer` and `status` (streaming task +message types), `lease` (the long-poll work-lease loop over core NATS), +`heartbeat`, `executor` and `session` (OpenJD session lifecycle and bare-metal +process execution), `isolation` (run-as-user execution, one Provider per GOOS +plus a fake), `pathmap` (storage-location path resolution), `staging` +(stage_locally copy in/out via the operator's sync command), `openjd` and +`fmtres` (worker-side OpenJD format-string and EXPR phase-3 resolution), +`envutil` (environment filtering), `logstreamer` and `status` (streaming task output and status back to the server), `cancel` (task/Session cancellation), +`diaglog` (the diagnostic-log sink that publishes to `worker.diag.`), `config`, `metrics`, and `obs` (observability). diff --git a/internal/api/openapi.yaml b/internal/api/openapi.yaml index f4f7fa1e..62e44d43 100644 --- a/internal/api/openapi.yaml +++ b/internal/api/openapi.yaml @@ -3660,17 +3660,19 @@ paths: | `type` | Description | |----------|-------------| - | `push` | Event for a subscribed subject. `seq` is the per-connection push counter. | + | `push` | Event for a subscribed subject. `seq` is the per-subject, hub-assigned push counter. | | `ack` | Acknowledges a client message. Payload: `{ "client_seq": N, "error": "" }`. | | `error` | Protocol-level error. Payload: `{ "code": "...", "message": "..." }`. | | `pong` | Reply to `ping`. No payload; `seq` is 0. | ## Sequence numbers and resumable subscriptions - Server push messages carry a `seq` that increments monotonically per - connection starting at 1. On reconnect, send `since_seq` in the - `subscribe` payload to replay any buffered events with `seq > since_seq`. - A `since_seq` of 0 (default) requests only live events. + Server push messages carry a `seq` assigned per subject by the hub, + starting at 1 and increasing globally across connections (not per + connection). Store the last `seq` you received and, on reconnect, send + it as `since_seq` in the `subscribe` payload to replay buffered events + with `seq > since_seq`. A `since_seq` of 0 (the default) replays every + buffered event for the subject, because hub sequences start at 1. ## Subject formats @@ -3680,11 +3682,19 @@ paths: | `jobs/{job_id}/tasks` | Task-level updates for the given job | | `workers` | Worker status changes | | `tasks/{task_id}/logs` | Log chunks for the given task attempt | - - ## Phase 1 note - - Authentication is not required in Phase 1. The `Authorization` header - hook is present but not enforced. This will change in Phase 3. + | `diagnostics` | Diagnostic log records from the server and every worker (requires `diagnostics.read`) | + + ## Authentication and per-subject authorization + + When `auth.enabled` is true the upgrade is authenticated exactly like a + REST request (session cookie or `Authorization: Bearer`), and a failure + is answered with an RFC-7807 `401` before the handshake. Subscribing to + `diagnostics` additionally requires the `diagnostics.read` permission, + and subscribing to `jobs/{job_id}/tasks` or `tasks/{task_id}/logs` + requires access to that job; a refusal is returned as a non-empty + `error` in the `ack` rather than as a connection close. With auth + disabled every connection is the anonymous superuser and nothing is + gated. parameters: - name: Upgrade in: header diff --git a/internal/config/config.go b/internal/config/config.go index e73d7b20..4b259f3d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -90,8 +90,10 @@ type StoreConfig struct { SQLitePath string `yaml:"sqlite_path"` // CheckpointInterval is how often the background goroutine runs a WAL - // checkpoint (PRAGMA wal_checkpoint(TRUNCATE)) to fold committed WAL frames + // checkpoint (PRAGMA wal_checkpoint(PASSIVE)) to fold committed WAL frames // back into the main database file and keep the WAL from growing unboundedly. + // The periodic checkpoint is PASSIVE so it never waits on a reader; the + // TRUNCATE checkpoint runs only on clean shutdown. // Must be > 0. Set to a large value (e.g. "24h") to effectively disable // periodic checkpointing (a final checkpoint always runs on shutdown). // Env: SQI_STORE_CHECKPOINT_INTERVAL diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 571d33f0..e8c74704 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -152,7 +152,7 @@ func New() *Metrics { Namespace: "sqi", Subsystem: "scheduler", Name: "queue_depth", - Help: "Current number of ready tasks waiting for assignment, partitioned by queue name.", + Help: "Current number of ready tasks waiting for assignment, partitioned by queue name. Excludes tasks in retry backoff, tasks on a paused queue, and tasks under a paused, parked or terminal job.", }, []string{"queue"}, ), diff --git a/internal/openjd/exprlimits.go b/internal/openjd/exprlimits.go index 1d0db74d..21047ee1 100644 --- a/internal/openjd/exprlimits.go +++ b/internal/openjd/exprlimits.go @@ -194,11 +194,13 @@ type ExprLimits struct { // The FLOORS all come from measurement, not from taste. The fourteen reference // presets in presets/sqi/ -- the templates this repo itself ships -- are // measured per dimension by binary search on every run, by -// TestExprLimits_FloorsAcceptReferencePresets: at most 31 expression positions, -// 1 retained byte, 390 live bytes and 15 operations in any single evaluation. -// Every one of those maxima is set by ffmpeg-segment-transcode-expr, the only -// preset doing real slice arithmetic; the render presets cost an order of -// magnitude less. Every floor below leaves at least 4x headroom over the worst +// TestExprLimits_FloorsAcceptReferencePresets: at most 31 expression positions +// template-wide, and 390 live bytes with 15 operations in any single +// evaluation. Those three maxima are set by ffmpeg-segment-transcode-expr, the +// only preset doing real slice arithmetic; the render presets cost an order of +// magnitude less. Retained bytes are effectively zero for all fourteen -- the +// binary search floors that dimension at 1, so a reported "1" means "nothing +// measurable", not a real cost. Every floor below leaves at least 4x headroom over the worst // of them (8x to 65536x in practice), so no operator can tighten a knob to a // value that rejects sqi's own templates. // TestExprLimits_FloorsAcceptReferencePresets re-measures that on every run @@ -225,14 +227,16 @@ const ( // any legal configuration the deadline, not this number, is what a request // actually stops at. // - // Floor: THREE orders of magnitude above what a reference preset's - // expressions actually spend, and one order of magnitude below the - // default so tightening remains meaningful. The measurement, re-run by + // Floor: one order of magnitude below the default so tightening remains + // meaningful, and ~67x above what a reference preset's expressions + // actually spend. The measurement, re-run by // TestExprLimits_FloorsAcceptReferencePresets on every test run, is that - // the most expensive single evaluation in any of the six presets costs - // exactly 1 operation -- so the headroom is 1000x. An earlier revision of - // this sentence said "two orders of magnitude above the handful of - // operations"; there is no handful, and the ratio was understated. + // the most expensive single evaluation in any of the fourteen presets + // costs 15 operations. Earlier revisions of this sentence said "two orders + // of magnitude above the handful of operations" and then "exactly 1 + // operation -- so the headroom is 1000x"; both were measured against a + // smaller preset set, and the ratio moves as presets are added. Trust the + // floors paragraph at the top of this file, which the test re-measures. MinExprSubmissionOperations int64 = 1_000 MaxExprSubmissionOperations int64 = 100_000 @@ -265,15 +269,15 @@ const ( // re-derive this ceiling from maxStringBytes; if the argument for 10x ever // changes, this number is free to move with it. // - // Floor: 4 KB, 64x the live bytes a reference preset's largest expression - // produces. That measurement -- 64 bytes, re-run by + // Floor: 4 KB, ~10x the live bytes a reference preset's largest expression + // produces. That measurement -- 390 bytes, re-run by // TestExprLimits_FloorsAcceptReferencePresets, and stated correctly in // this file's own floors paragraph above -- is what the number is sized - // from. An earlier revision of this sentence said "three orders of - // magnitude above the few hundred live bytes": both halves were wrong - // (the cost is 64 bytes, not a few hundred, and the ratio is ~1.8 orders, - // not 3), and it survived the fix round that added the correct paragraph - // three screens up. + // from. Earlier revisions of this sentence said "three orders of magnitude + // above the few hundred live bytes" and then "64 bytes ... ~1.8 orders"; + // both were measured against a smaller preset set, and the figure moves as + // presets are added. Trust the floors paragraph three screens up, which + // the test re-measures. MinExprSubmissionMemoryBytes int64 = 4_096 MaxExprSubmissionMemoryBytes int64 = 10_000_000 @@ -286,7 +290,7 @@ const ( // multiplies that derived number by 100 (10^8 -> 10^10). Documentation // must say so in those words; see design spec §4 caveat 1. // - // Floor: 256, roughly 17x the largest reference preset's measured 15 + // Floor: 256, roughly 8x the largest reference preset's measured 31 // positions, and still ~40x below the default so an operator running only // small templates can tighten hard. MinExprTemplatePositions int64 = 256 diff --git a/internal/product/sqipresets_test.go b/internal/product/sqipresets_test.go index 686622b8..300c63a5 100644 --- a/internal/product/sqipresets_test.go +++ b/internal/product/sqipresets_test.go @@ -47,12 +47,12 @@ func TestSQIReferencePresets(t *testing.T) { "maya-layer-render": { category: "Rendering", params: []string{"SceneFile", "Frames", "OutputDir", "Renderer", "RenderLayer"}, - extensions: []string{"TASK_CHUNKING"}, + extensions: []string{"TASK_CHUNKING", "SQI_CHUNK_BOUNDS"}, }, "maya-scene-render": { category: "Rendering", params: []string{"SceneFile", "Frames", "OutputDir", "Renderer"}, - extensions: []string{"TASK_CHUNKING"}, + extensions: []string{"TASK_CHUNKING", "SQI_CHUNK_BOUNDS"}, }, "houdini-rop-render": { category: "Rendering", @@ -72,11 +72,14 @@ func TestSQIReferencePresets(t *testing.T) { "blender-batch-render": { category: "Rendering", params: []string{"SceneFile", "Frames", "OutputPath"}, - extensions: []string{"TASK_CHUNKING"}, + extensions: []string{"TASK_CHUNKING", "SQI_CHUNK_BOUNDS"}, }, - // The three Mistika presets are the only shipped presets that declare - // SQI_CHUNK_BOUNDS: they invoke the renderer as `-s START -e END`, so - // they need a chunk's bounds rather than the chunk itself. + // The three Mistika presets invoke the renderer as `-s START -e END`, + // so they need a chunk's bounds rather than the chunk itself. They are + // not alone in that: the two Maya presets and blender-batch-render + // declare SQI_CHUNK_BOUNDS for the same reason, so six of the nine + // render presets use it. Only Houdini and the two Nuke presets take a + // frame list directly and can do without it. "mistika-boutique-render": { category: "Rendering", params: []string{"SceneFile", "Frames"}, diff --git a/internal/store/statemachine.go b/internal/store/statemachine.go index ee47d1a9..7d873178 100644 --- a/internal/store/statemachine.go +++ b/internal/store/statemachine.go @@ -80,8 +80,11 @@ var validTaskTransitions = map[TaskStatus]map[TaskStatus]struct{}{ // // This lives in package store rather than package openjd because the store is // what enforces it on every write; openjd imports store, so the store cannot -// import openjd. [github.com/uberware/sqi/internal/openjd.ValidateTaskTransition] -// delegates here. +// import openjd. There is no openjd counterpart to call instead: package openjd +// owns only the STEP machine ([github.com/uberware/sqi/internal/openjd.ValidateStepTransition]), +// with its own separate [ErrInvalidTransition] sentinel — match errors against +// the sentinel from the same package as the machine you called, or errors.Is +// silently stops matching. func ValidateTaskTransition(from, to TaskStatus) error { targets, known := validTaskTransitions[from] if !known { diff --git a/pkg/README.md b/pkg/README.md index 6b24e599..4ceb11ef 100644 --- a/pkg/README.md +++ b/pkg/README.md @@ -4,4 +4,6 @@ Public Go packages — importable by external code (the `sqi-sdk` Python wheel i Anything placed here is part of the public API contract and must be versioned accordingly. Internal-only code belongs in `internal/`. -This directory is empty in Phase 1; populated as exported surfaces stabilize. +No public packages have been promoted here yet — `doc.go` only carries the +package documentation that reserves the directory. It is populated as exported +surfaces stabilize. diff --git a/pkg/doc.go b/pkg/doc.go index 18fc4a29..f3a104cf 100644 --- a/pkg/doc.go +++ b/pkg/doc.go @@ -2,12 +2,12 @@ // Package pkg is the root of the sqi public Go API surface. // -// # Phase 1 status +// # Status // -// This package tree is intentionally empty in Phase 1. All sqi-server -// internals live under internal/ and are not importable by external code. -// The pkg/ tree will be populated as surfaces stabilize and are ready to -// carry a compatibility commitment. +// No public packages have been promoted here yet. All sqi-server internals +// live under internal/ and are not importable by external code. The pkg/ +// tree will be populated as surfaces stabilize and are ready to carry a +// compatibility commitment. // // # Planned packages // diff --git a/scripts/README.md b/scripts/README.md index 5e1c96a7..fc7addd6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,5 +1,14 @@ # `scripts/` -Developer and CI shell scripts: local bootstrap, release helpers, schema regeneration, lint shortcuts, and anything else that is easier as a script than a Makefile target. +Developer and CI scripts that are easier to express as a script than as a Makefile recipe: -Scripts here should be POSIX-portable where reasonable (`#!/usr/bin/env bash`, `set -euo pipefail`) and document their inputs at the top. +- `smoke.sh` — end-to-end smoke test against the built binaries (REST + WebSocket + an EXPR job); run it via `make smoke` +- `auth-demo.sh` — drives the auth surface against a live local farm; run it via `make auth-demo` (`KEEP=1` leaves the farm running) +- `macos-sign.sh` — Developer ID codesigning hook invoked by goreleaser's post-build hooks; a no-op for non-darwin targets and for snapshot builds +- `expr-oracle.py` — feeds `test/oracle/corpus.txt` to the pinned OpenJD reference implementation over JSON lines for `make test-expr-oracle` +- `test-isolation-windows.ps1` — runs the Windows run-as-user isolation suite (tier 1 as the elevated admin, tier 2 re-launched as SYSTEM via a scheduled task); run it via `make test-isolation-windows` from an **elevated** shell + +Shell scripts here should be POSIX-portable where reasonable +(`#!/usr/bin/env bash`, `set -euo pipefail`) and document their inputs at the +top. The PowerShell and Python entries above are deliberate exceptions — +Windows-only and reference-implementation glue respectively. diff --git a/test/README.md b/test/README.md index 89c61ba9..0be181b7 100644 --- a/test/README.md +++ b/test/README.md @@ -2,13 +2,15 @@ Cross-cutting test assets that do not belong next to a single Go package: -- `test/integration/` — end-to-end harness that boots the server with a temp SQLite + embedded NATS, registers a mock worker, and runs a real OpenJD job -- `test/load/` — scheduler throughput and assignment-latency benchmarks -- `test/fixtures/openjd/` — corpus of valid and invalid OpenJD templates used by parser tests and fuzzers -- `test/smoke/` — end-to-end smoke script for the release verification step +- `test/integration/` — end-to-end harness that boots the server with a temp SQLite + embedded NATS, registers a mock worker, and runs a real OpenJD job; also holds the real-directory (LDAP), real-provider (OIDC) and real-root (run-as-user isolation) suites +- `test/conformance/` — the official OpenJD conformance suite run against the vendored `third_party/` fixtures (build tag `conformance`; `make test-conformance`) +- `test/oracle/` — the EXPR differential test against the OpenJD reference implementation, plus its corpus and adjudicated baseline (build tag `oracle`; `make test-expr-oracle`) Unit tests live next to the code they cover (`_test.go` files), not here. +The end-to-end smoke script used by release verification is `scripts/smoke.sh` +(run it via `make smoke`), not a directory under `test/`. + ## Build tags Most files in `test/integration/` are untagged and run with `make test`. Files diff --git a/web/README.md b/web/README.md index e5082f30..0a36c044 100644 --- a/web/README.md +++ b/web/README.md @@ -26,7 +26,7 @@ web/ ├── routes.tsx # React Router route table ├── api/ # Typed REST client, domain types, TanStack Query hooks ├── ws/ # WebSocket client, React context, typed event payloads - ├── components/ # Reusable UI components (DataTable, StatusBadge, …) + ├── components/ # Reusable UI components (StatusBadge, Pagination, LogViewer, …) ├── pages/ # Top-level views (Dashboard, JobList, Submit, …) ├── hooks/ # Shared hooks (useDebounce, usePaginatedList, …) ├── styles/ # Design tokens and global CSS @@ -45,7 +45,7 @@ mirrors the REST wire format (kept in sync with `GET /api/v1/openapi.yaml`), and Install dependencies once: ```sh -npm install +npm ci ``` ## Local development From 8488b80c39257ed5a4b6ab9282244437abef50a4 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Mon, 17 Aug 2026 22:56:30 -0700 Subject: [PATCH 02/10] chore(release): bump version references to 0.3.0 --- clients/python/src/sqi_client/_version.py | 2 +- clients/submitter/src/sqi_submitter/_version.py | 2 +- deploy/docker-compose.yml | 4 ++-- docs/examples/hello.json | 2 +- docs/operations.md | 2 +- docs/quickstart.md | 2 +- docs/release-runbook.md | 4 ++-- docs/worker-docker.md | 4 ++-- internal/api/openapi.yaml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/clients/python/src/sqi_client/_version.py b/clients/python/src/sqi_client/_version.py index cd0d4216..5b7ce2c5 100644 --- a/clients/python/src/sqi_client/_version.py +++ b/clients/python/src/sqi_client/_version.py @@ -9,4 +9,4 @@ from __future__ import annotations -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/clients/submitter/src/sqi_submitter/_version.py b/clients/submitter/src/sqi_submitter/_version.py index 374297ec..0b20e7c9 100644 --- a/clients/submitter/src/sqi_submitter/_version.py +++ b/clients/submitter/src/sqi_submitter/_version.py @@ -1,4 +1,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later """Single-source package version (read by hatchling and the package).""" -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index c996c950..23bd63e4 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -12,7 +12,7 @@ services: sqi-server: - image: ghcr.io/uberware/sqi/sqi-server:v0.2.0 + image: ghcr.io/uberware/sqi/sqi-server:v0.3.0 ports: - "8080:8080" # REST API + web UI - "4222:4222" # embedded NATS (workers connect here) @@ -29,7 +29,7 @@ services: - server-data:/var/lib/sqi-server sqi-worker: - image: ghcr.io/uberware/sqi/sqi-worker:v0.2.0 + image: ghcr.io/uberware/sqi/sqi-worker:v0.3.0 depends_on: - sqi-server restart: unless-stopped diff --git a/docs/examples/hello.json b/docs/examples/hello.json index 1c1aaf04..b48126c5 100644 --- a/docs/examples/hello.json +++ b/docs/examples/hello.json @@ -8,7 +8,7 @@ "actions": { "onRun": { "command": "echo", - "args": ["Hello from sqi v0.2.0"] + "args": ["Hello from sqi v0.3.0"] } } } diff --git a/docs/operations.md b/docs/operations.md index cb1e2d7c..68f10aa2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -488,5 +488,5 @@ sqi-server config print --config /etc/sqi/sqi-server.yaml ```sh sqi-server version -# sqi-server v0.2.0 (commit abc1234, built 2026-07-09, go1.26.3) +# sqi-server v0.3.0 (commit abc1234, built 2026-07-09, go1.26.3) ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 6023f1e9..24f832db 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -33,7 +33,7 @@ http://localhost:8080. ## Option B — Docker Compose ```sh -curl -LO https://raw.githubusercontent.com/uberware/sqi/v0.2.0/deploy/docker-compose.yml +curl -LO https://raw.githubusercontent.com/uberware/sqi/v0.3.0/deploy/docker-compose.yml docker compose -f docker-compose.yml up -d ``` diff --git a/docs/release-runbook.md b/docs/release-runbook.md index 4d596036..1b955986 100644 --- a/docs/release-runbook.md +++ b/docs/release-runbook.md @@ -190,10 +190,10 @@ image, plus the manifest tags). ## Step 6: Cut the real release Set the target version once and reuse it throughout this section (examples below -use `0.2.0`): +use `0.3.0`): ```bash -VERSION=0.2.0 +VERSION=0.3.0 ``` **Procedure:** diff --git a/docs/worker-docker.md b/docs/worker-docker.md index fe56ee5c..b6f5e41b 100644 --- a/docs/worker-docker.md +++ b/docs/worker-docker.md @@ -17,10 +17,10 @@ ghcr.io/uberware/sqi/sqi-worker: | Tag | Description | |---|---| | `latest` | Rolling build of the `main` branch — republished on every push to `main` (may be ahead of the last tagged release) | -| `v0.2.0` | A specific tagged release | +| `v0.3.0` | A specific tagged release | | `v0` | Rolling major-version tag — the newest `v0.x` release | -For production, pin a specific release tag (`v0.2.0`) for reproducibility. +For production, pin a specific release tag (`v0.3.0`) for reproducibility. There is no `main` tag; the `latest` tag tracks the `main` branch. Pull the image: diff --git a/internal/api/openapi.yaml b/internal/api/openapi.yaml index 62e44d43..94e652f6 100644 --- a/internal/api/openapi.yaml +++ b/internal/api/openapi.yaml @@ -44,7 +44,7 @@ info: not unauthenticated either: it runs its own upgrade hook (a valid principal is still required) plus a subject-level `diagnostics.read` check before allowing a subscription to the diagnostics feed. - version: "0.2.0" + version: "0.3.0" license: name: AGPL-3.0-or-later url: https://www.gnu.org/licenses/agpl-3.0.html From 7d2e5348d44a5b53e29003b2a6328f5c1e09a043 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Mon, 17 Aug 2026 22:56:30 -0700 Subject: [PATCH 03/10] docs: mark phase 3 and the EXPR extension as released in v0.3.0 --- CONTRIBUTING.md | 2 +- README.md | 4 ++-- ROADMAP.md | 6 +++--- docs/contributing.md | 2 +- docs/index.md | 4 ++-- docs/roadmap.md | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f745344..3e284b77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -214,7 +214,7 @@ The [ROADMAP.md](ROADMAP.md) document outlines development phases. Current prior - **Phase 1** (v0.1 — released): Core scheduler, pull-based workers, basic web UI, OpenJD execution - **Phase 2** (v0.2 — released): Product system, preset library, DCC submitters -- **Phase 3** (complete, unreleased): Auth (local accounts, API keys, RBAC, LDAP/AD, OAuth2/OIDC SSO), multi-user role model, and run-as-user task isolation +- **Phase 3** (v0.3 — released): Auth (local accounts, API keys, RBAC, LDAP/AD, OAuth2/OIDC SSO), multi-user role model, and run-as-user task isolation; v0.3 also ships the OpenJD `EXPR` extension and expanded ffmpeg and Mistika reference presets - **Phase 4** (next): Production hardening, PostgreSQL, HA, auto-scaling Code contributions aligned with the current phase are most likely to be accepted quickly. Contributions targeting later phases are welcome but may take longer to review if they require design discussion. diff --git a/README.md b/README.md index 0a4740a2..bb3265e9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ 📖 **[Documentation](https://uberware.github.io/sqi/)** — quickstart, architecture, configuration, and full reference. -> **Status:** v0.2.0 (Phase 2) released. Builds on the Phase 1 core with products and presets as an authoring layer over OpenJD, a community preset-library integration, product-driven submission, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** (opt-in auth & multi-user) — local accounts, API keys, RBAC, job-owner identity, LDAP/AD, OAuth2/OIDC SSO, and run-as-user task isolation — is complete and merged on `main`, unreleased. **Phase 4** (production hardening) is next. Contributions, feedback, and discussion welcome. +> **Status:** v0.3.0 (Phase 3) released — opt-in authentication and multi-user support: local accounts, API keys, RBAC, job-owner identity, LDAP/AD, OAuth2/OIDC SSO, and queue-scoped run-as-user task isolation on POSIX and Windows. Also in this release: the official OpenJD `EXPR` expression-language extension, and expanded reference presets (ffmpeg transcode, sequence-encode and segment-transcode; Mistika Boutique, VR and Workflows). Auth is **off by default**, so an existing deployment is unaffected until enabled. **Phase 4** (production hardening) is next. Contributions, feedback, and discussion welcome. --- @@ -81,7 +81,7 @@ Both modes run the same software. The difference is configuration. ## Status and roadmap -`sqi` v0.1.0 delivered the Phase 1 core: scheduler, pull-based workers, OpenJD job execution, and a basic web UI, with a Python client and single-binary or Docker Compose deployment. v0.2.0 completes Phase 2 — products and presets as an authoring layer over OpenJD, the community preset-library integration, a product-driven submission form, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** — opt-in authentication and multi-user support: local accounts and API keys, role-based access control, an authenticated owner/submitter identity on jobs, LDAP/Active Directory, OAuth2/OIDC SSO, and queue-scoped run-as-user task isolation (tasks execute as a distinct, unprivileged OS account instead of the worker's own, on POSIX and Windows) — is complete and merged on `main`, unreleased. Auth is off by default, so an existing deployment is unaffected until enabled. Production-hardening features (Phase 4) follow. +`sqi` v0.1.0 delivered the Phase 1 core: scheduler, pull-based workers, OpenJD job execution, and a basic web UI, with a Python client and single-binary or Docker Compose deployment. v0.2.0 completes Phase 2 — products and presets as an authoring layer over OpenJD, the community preset-library integration, a product-driven submission form, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** — opt-in authentication and multi-user support: local accounts and API keys, role-based access control, an authenticated owner/submitter identity on jobs, LDAP/Active Directory, OAuth2/OIDC SSO, and queue-scoped run-as-user task isolation (tasks execute as a distinct, unprivileged OS account instead of the worker's own, on POSIX and Windows) — ships in v0.3.0, alongside the official OpenJD `EXPR` expression-language extension and expanded ffmpeg and Mistika reference presets. Auth is off by default, so an existing deployment is unaffected until enabled. Production-hardening features (Phase 4) follow. This is a real project with a concrete development commitment, not a design document waiting for funding. Feedback on priorities is welcome — [open an issue](https://github.com/uberware/sqi/issues/new) or [start a discussion](https://github.com/uberware/sqi/discussions/new/choose). diff --git a/ROADMAP.md b/ROADMAP.md index db4e71b2..d53c8dcf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # `sqi` Roadmap and Technical Architecture -> **Status:** v0.2.0 (Phase 2) released. Phase 3 (auth and multi-user) — including task isolation (run-as-user) — is complete and merged on `main`, unreleased ahead of the v0.3 release. Details may change with requests, feedback, and discoveries as development progresses. +> **Status:** v0.3.0 (Phase 3) released — opt-in authentication and multi-user support, including task isolation (run-as-user), plus the official OpenJD `EXPR` extension and expanded ffmpeg and Mistika reference presets. **Phase 4** (production hardening) is next. Details may change with requests, feedback, and discoveries as development progresses. This document provides technical detail on `sqi`'s architecture, core concepts, and development roadmap. For the vision and feature overview, see [README.md](README.md). @@ -250,7 +250,7 @@ NATS can run embedded within `sqi-server` (simple mode) or as a separate cluster - Cross-job dependencies — a submission may declare `depends_on` upstream jobs (same farm, across queues); dependents are held `blocked` until every upstream completes, then released (or canceled if an upstream fails) - Testing job presets — ready-to-run `test-render`/`test-steps` presets (bash and PowerShell) published to the preset library for smoke-testing a farm -### Phase 3: Auth and Multi-User (v0.3) — complete, unreleased +### Phase 3: Auth and Multi-User (v0.3) ✅ Released Authentication is **opt-in and off by default** — an unconfigured server behaves exactly as it did before Phase 3. See [docs/auth.md](docs/auth.md) for the model and setup. @@ -271,7 +271,7 @@ before Phase 3. See [docs/auth.md](docs/auth.md) for the model and setup. [docs/worker-configuration.md](docs/worker-configuration.md) for the model, setup, and known gaps. -### Post-Phase-3, pre-v0.3: OpenJD `EXPR` and expanded reference presets — complete, unreleased +### Also in v0.3: OpenJD `EXPR` and expanded reference presets ✅ Released - **OpenJD `EXPR` extension** — the official expression-language extension is fully implemented and `StatusSupported`: expression core, type system, diff --git a/docs/contributing.md b/docs/contributing.md index 07d57458..5c572b78 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -214,7 +214,7 @@ The [ROADMAP.md](roadmap.md) document outlines development phases. Current prior - **Phase 1** (v0.1 — released): Core scheduler, pull-based workers, basic web UI, OpenJD execution - **Phase 2** (v0.2 — released): Product system, preset library, DCC submitters -- **Phase 3** (complete, unreleased): Auth (local accounts, API keys, RBAC, LDAP/AD, OAuth2/OIDC SSO), multi-user role model, and run-as-user task isolation +- **Phase 3** (v0.3 — released): Auth (local accounts, API keys, RBAC, LDAP/AD, OAuth2/OIDC SSO), multi-user role model, and run-as-user task isolation; v0.3 also ships the OpenJD `EXPR` extension and expanded ffmpeg and Mistika reference presets - **Phase 4** (next): Production hardening, PostgreSQL, HA, auto-scaling Code contributions aligned with the current phase are most likely to be accepted quickly. Contributions targeting later phases are welcome but may take longer to review if they require design discussion. diff --git a/docs/index.md b/docs/index.md index e8b98540..3efed63e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ `sqi` (pronounced "sky") is an open source distributed task and render farm manager built for modern production pipelines. It is designed to run simply on a handful of local workstations and scale to hybrid on-premises and cloud infrastructure without changing how you work. -> **Status:** v0.2.0 (Phase 2) released. Builds on the Phase 1 core with products and presets as an authoring layer over OpenJD, a community preset-library integration, product-driven submission, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** (opt-in auth & multi-user) — local accounts, API keys, RBAC, job-owner identity, LDAP/AD, OAuth2/OIDC SSO, and run-as-user task isolation — is complete and merged on `main`, unreleased. **Phase 4** (production hardening) is next. Contributions, feedback, and discussion welcome. +> **Status:** v0.3.0 (Phase 3) released — opt-in authentication and multi-user support: local accounts, API keys, RBAC, job-owner identity, LDAP/AD, OAuth2/OIDC SSO, and queue-scoped run-as-user task isolation on POSIX and Windows. Also in this release: the official OpenJD `EXPR` expression-language extension, and expanded reference presets (ffmpeg transcode, sequence-encode and segment-transcode; Mistika Boutique, VR and Workflows). Auth is **off by default**, so an existing deployment is unaffected until enabled. **Phase 4** (production hardening) is next. Contributions, feedback, and discussion welcome. --- @@ -79,7 +79,7 @@ Both modes run the same software. The difference is configuration. ## Status and roadmap -`sqi` v0.1.0 delivered the Phase 1 core: scheduler, pull-based workers, OpenJD job execution, and a basic web UI, with a Python client and single-binary or Docker Compose deployment. v0.2.0 completes Phase 2 — products and presets as an authoring layer over OpenJD, the community preset-library integration, a product-driven submission form, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** — opt-in authentication and multi-user support: local accounts and API keys, role-based access control, an authenticated owner/submitter identity on jobs, LDAP/Active Directory, OAuth2/OIDC SSO, and queue-scoped run-as-user task isolation (tasks execute as a distinct, unprivileged OS account instead of the worker's own, on POSIX and Windows) — is complete and merged on `main`, unreleased. Auth is off by default, so an existing deployment is unaffected until enabled. Production-hardening features (Phase 4) follow. +`sqi` v0.1.0 delivered the Phase 1 core: scheduler, pull-based workers, OpenJD job execution, and a basic web UI, with a Python client and single-binary or Docker Compose deployment. v0.2.0 completes Phase 2 — products and presets as an authoring layer over OpenJD, the community preset-library integration, a product-driven submission form, additional path-translation modes, S3-compatible storage, compute locations, and in-application DCC submitters for Maya, Houdini, Nuke, and Blender. **Phase 3** — opt-in authentication and multi-user support: local accounts and API keys, role-based access control, an authenticated owner/submitter identity on jobs, LDAP/Active Directory, OAuth2/OIDC SSO, and queue-scoped run-as-user task isolation (tasks execute as a distinct, unprivileged OS account instead of the worker's own, on POSIX and Windows) — ships in v0.3.0, alongside the official OpenJD `EXPR` expression-language extension and expanded ffmpeg and Mistika reference presets. Auth is off by default, so an existing deployment is unaffected until enabled. Production-hardening features (Phase 4) follow. This is a real project with a concrete development commitment, not a design document waiting for funding. Feedback on priorities is welcome — [open an issue](https://github.com/uberware/sqi/issues/new) or [start a discussion](https://github.com/uberware/sqi/discussions/new/choose). diff --git a/docs/roadmap.md b/docs/roadmap.md index 6218a026..72281231 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # `sqi` Roadmap and Technical Architecture -> **Status:** v0.2.0 (Phase 2) released. Phase 3 (auth and multi-user) — including task isolation (run-as-user) — is complete and merged on `main`, unreleased ahead of the v0.3 release. Details may change with requests, feedback, and discoveries as development progresses. +> **Status:** v0.3.0 (Phase 3) released — opt-in authentication and multi-user support, including task isolation (run-as-user), plus the official OpenJD `EXPR` extension and expanded ffmpeg and Mistika reference presets. **Phase 4** (production hardening) is next. Details may change with requests, feedback, and discoveries as development progresses. This document provides technical detail on `sqi`'s architecture, core concepts, and development roadmap. For the vision and feature overview, see [README.md](index.md). @@ -250,7 +250,7 @@ NATS can run embedded within `sqi-server` (simple mode) or as a separate cluster - Cross-job dependencies — a submission may declare `depends_on` upstream jobs (same farm, across queues); dependents are held `blocked` until every upstream completes, then released (or canceled if an upstream fails) - Testing job presets — ready-to-run `test-render`/`test-steps` presets (bash and PowerShell) published to the preset library for smoke-testing a farm -### Phase 3: Auth and Multi-User (v0.3) — complete, unreleased +### Phase 3: Auth and Multi-User (v0.3) ✅ Released Authentication is **opt-in and off by default** — an unconfigured server behaves exactly as it did before Phase 3. See [docs/auth.md](auth.md) for the model and setup. @@ -271,7 +271,7 @@ before Phase 3. See [docs/auth.md](auth.md) for the model and setup. [docs/worker-configuration.md](worker-configuration.md) for the model, setup, and known gaps. -### Post-Phase-3, pre-v0.3: OpenJD `EXPR` and expanded reference presets — complete, unreleased +### Also in v0.3: OpenJD `EXPR` and expanded reference presets ✅ Released - **OpenJD `EXPR` extension** — the official expression-language extension is fully implemented and `StatusSupported`: expression core, type system, From b23d35e8cfd4416a9d7ff8a76694738ceb901e30 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Mon, 17 Aug 2026 23:04:49 -0700 Subject: [PATCH 04/10] build: replace deprecated goreleaser archives keys so goreleaser check passes --- .goreleaser.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 2d886f31..5bbfc095 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -108,7 +108,7 @@ builds: archives: - id: sqi - builds: + ids: - sqi-server - sqi-worker name_template: >- @@ -118,7 +118,7 @@ archives: {{- else }}{{ .Arch }}{{ end }} format_overrides: - goos: windows - format: zip + formats: [zip] files: - LICENSE - README.md From b17cca30bdca9a23dd7c41d634ecd96aa22a1e96 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Mon, 17 Aug 2026 23:05:03 -0700 Subject: [PATCH 05/10] chore(release): update changelog for 0.3.0 --- CHANGELOG.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3da84e07..1ba178a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,47 @@ Format follows [Conventional Commits](https://www.conventionalcommits.org/) and > `docs/development.md`). It is refreshed during release prep and again by the > release workflow, which bundles it into the release archives. +## [0.3.0] — 2026-08-18 + + +### Bug Fixes + +- Duplicate websocket push on subscribe ([#87](https://github.com/uberware/sqi/issues/87)) ([19b0607](https://github.com/uberware/sqi/commit/19b060770ff25a67dc94b2172e4b570a3353e008)) +- Enforce task state machine on task status writes ([#97](https://github.com/uberware/sqi/issues/97)) ([9c386d4](https://github.com/uberware/sqi/commit/9c386d42c6f28a6ab9250a53916063fc1242b4c9)) +- **web:** Upgrade react-router to 8.3.0 for GHSA-qwww-vcr4-c8h2 ([#101](https://github.com/uberware/sqi/issues/101)) ([09b3ac6](https://github.com/uberware/sqi/commit/09b3ac636b22f0d15674ee557157a9c8bdfef0c6)) +- Atomic job submission ([#113](https://github.com/uberware/sqi/issues/113)) ([b1aa609](https://github.com/uberware/sqi/commit/b1aa609273526d11c92cd516934fbb6b6cf7b3fc)) +- **scheduler:** Match attr.worker.os.family macos against a darwin worker ([#116](https://github.com/uberware/sqi/issues/116)) ([cdf0815](https://github.com/uberware/sqi/commit/cdf0815ef964a250d426d170855e566ba64f4f0a)) +- **scheduler:** Resolve attr.worker.cpu.arch against a worker-reporte… ([#117](https://github.com/uberware/sqi/issues/117)) ([874c85f](https://github.com/uberware/sqi/commit/874c85fa29e22afd89a3eb447e1c18f97bc5aea3)) + + +### Build + +- Replace deprecated goreleaser archives keys so goreleaser check passes ([c5d75de](https://github.com/uberware/sqi/commit/c5d75de68e8bc7df23c5fd0c11e8e8529731d6c4)) + + +### Documentation + +- Correct project status and documentation drift ([#98](https://github.com/uberware/sqi/issues/98)) ([9026e68](https://github.com/uberware/sqi/commit/9026e68cba99bbb3cad386440c642d3b22e463af)) +- Correct inaccurate and stale documentation across the repo ([5ac8fc4](https://github.com/uberware/sqi/commit/5ac8fc4a7aefd236ce52c8e7c22cd2c14ed27a18)) +- Mark phase 3 and the EXPR extension as released in v0.3.0 ([8d6c278](https://github.com/uberware/sqi/commit/8d6c2782cc6b35ad6e003db6a9e641fbf664d263)) + + +### Features + +- Authentication infrastructure ([#80](https://github.com/uberware/sqi/issues/80)) ([15e72b6](https://github.com/uberware/sqi/commit/15e72b66037dca9c38fe108eeb91fe499d1bf7ae)) +- Local accounts, login sessions, auth shell ([#81](https://github.com/uberware/sqi/issues/81)) ([a55e203](https://github.com/uberware/sqi/commit/a55e203cb3852c4f849d918853cc2bfb45b258e1)) +- Api keys ([#82](https://github.com/uberware/sqi/issues/82)) ([7dcb0e0](https://github.com/uberware/sqi/commit/7dcb0e0dc49c043c517e09c81b699084f529aa7e)) +- Role-based access control ([#83](https://github.com/uberware/sqi/issues/83)) ([3190510](https://github.com/uberware/sqi/commit/3190510f14c6055d5759ed33dd2de7b5f5058918)) +- Job owner binding ([#84](https://github.com/uberware/sqi/issues/84)) ([8d7cf58](https://github.com/uberware/sqi/commit/8d7cf58c5c2b603eb31e2d9b6b4afb1b7167b387)) +- Auth admin and testing ([#85](https://github.com/uberware/sqi/issues/85)) ([e1003da](https://github.com/uberware/sqi/commit/e1003daa5cdaf98acd8c4db12b1400894e1b52f4)) +- LDAP and AD integration ([#86](https://github.com/uberware/sqi/issues/86)) ([4339e31](https://github.com/uberware/sqi/commit/4339e31bd086d0fcd3451dc38917b00662b5dc19)) +- OIDC compatible SSO and standardization with LDAP support ([#95](https://github.com/uberware/sqi/issues/95)) ([842e198](https://github.com/uberware/sqi/commit/842e198031515da35df286bb5c7e4f860394a977)) +- Task isolation run as user ([#99](https://github.com/uberware/sqi/issues/99)) ([785034d](https://github.com/uberware/sqi/commit/785034d7ece38ef23398f88cc05796c91c1cc839)) +- Improved openJD conformance ([#100](https://github.com/uberware/sqi/issues/100)) ([2cdef4f](https://github.com/uberware/sqi/commit/2cdef4f78618cfdabad999d7559180c4a31ba585)) +- Support the OpenJD EXPR extension ([#114](https://github.com/uberware/sqi/issues/114)) ([0682598](https://github.com/uberware/sqi/commit/068259857ae4c3557f47ebade8b17da11b33f402)) +- Ffmpeg presets ([#115](https://github.com/uberware/sqi/issues/115)) ([91bb943](https://github.com/uberware/sqi/commit/91bb943281a5cc04c78220fc91819330e2eec364)) +- Mistika render presets ([#118](https://github.com/uberware/sqi/issues/118)) ([fe14c99](https://github.com/uberware/sqi/commit/fe14c99ead5acc63e93bb29b9e59f20b19ba7b99)) + ## [0.2.0] — 2026-07-13 From 835636d8e25853f5c9afd76f6047226df5ce4047 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Tue, 18 Aug 2026 11:17:43 -0700 Subject: [PATCH 06/10] feat(presets): trim descriptions and drop the bash variant's OS gate --- .github/workflows/ci.yml | 27 ++++++++++++++----- docs/openjd-extensions/expr.md | 8 +++--- docs/preset-library.md | 20 +++++++++----- docs/products.md | 5 ++-- presets/sqi/blender-batch-render.yaml | 3 +-- .../sqi/ffmpeg-segment-transcode-bash.yaml | 12 ++++----- presets/sqi/ffmpeg-sequence-encode.yaml | 8 ++---- presets/sqi/ffmpeg-transcode.yaml | 6 +---- presets/sqi/houdini-rop-render.yaml | 3 +-- presets/sqi/maya-layer-render.yaml | 3 +-- presets/sqi/maya-scene-render.yaml | 3 +-- presets/sqi/mistika-boutique-render.yaml | 3 +-- presets/sqi/mistika-vr-render.yaml | 3 +-- presets/sqi/mistika-workflows-render.yaml | 3 +-- presets/sqi/nuke-script-render.yaml | 3 +-- presets/sqi/nuke-write-render.yaml | 3 +-- test/integration/ffmpeg_presets_test.go | 21 +++++++++++---- 17 files changed, 75 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3aa1a334..a28e6298 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1009,9 +1009,12 @@ jobs: # the same guard conformance, expr-oracle, ldap-integration and # isolation-integration use. # - # Linux only. The bash-joined variant gates on a linux worker and can run - # NOWHERE else; the Windows-gated variant has its own job below - # (ffmpeg-presets-windows), because it can run nowhere but a Windows runner. + # Linux is where the bash-joined variant's coverage is REQUIRED by name below. + # That variant no longer gates on an OS -- bash is not POSIX-only, git-bash + # puts it on Windows too -- so it also runs on the Windows job, but only + # opportunistically there (see that job's own note). The PowerShell-gated + # variant has its own job below (ffmpeg-presets-windows), because it can run + # nowhere but a Windows runner. ffmpeg-presets: name: ffmpeg reference presets (execution) runs-on: ubuntu-latest @@ -1104,10 +1107,20 @@ jobs: go test -run TestFFmpegPreset -v -timeout 900s ./test/integration/ 2>&1 \ | tee /tmp/ffmpeg-presets-windows.log - # Only the PowerShell case is required by name. The rest either skip here - # (the bash variant gates on linux) or are already required by name on the - # Linux job, and duplicating them would make this job fail for reasons - # that have nothing to do with Windows. + # Only the PowerShell case is required by name. The rest are already + # required by name on the Linux job, and duplicating them would make this + # job fail for reasons that have nothing to do with Windows. + # + # TestFFmpegPreset_BashSegmentTranscodeJoins is the interesting omission. + # Its preset dropped its linux/macos gate (git-bash means bash is not + # POSIX-only), so it no longer skips here by construction -- it runs, and + # if its join fails on Windows the `go test` step above goes red, which is + # exactly the coverage that change needs. It is NOT required by name yet + # because it skips when `bash.exe` is not resolvable by Windows PATH + # lookup from the worker process, and that has not been observed on this + # runner image: requiring it now would turn the job red for an + # environmental reason rather than a real one. Promote it to the list once + # a run here is seen to PASS rather than SKIP. - name: Assert the PowerShell join actually ran shell: bash run: | diff --git a/docs/openjd-extensions/expr.md b/docs/openjd-extensions/expr.md index 60fd0f22..8457aa1f 100644 --- a/docs/openjd-extensions/expr.md +++ b/docs/openjd-extensions/expr.md @@ -25,15 +25,17 @@ transcode, all three of which declare `extensions: [EXPR]` (`presets/sqi/ffmpeg-sequence-encode.yaml` declares it too, more modestly). One, `ffmpeg-segment-transcode-expr` ("Portable"), builds its ffmpeg concat file list from an EXPR comprehension instead of a join-time shell script, so it -runs on Linux, macOS, and Windows workers alike with no shell at all. That +needs no shell on the worker at all — not even `bash`, which the bash-joined +variant needs but is likewise not gated to any one OS. That list-building expression references only job parameters, so it is fully resolved at **submission** (phase 2, not phase 3) and its cost is charged against the *server's* [`openjd.expr_operation_limit`](../configuration.md#openjdexpr_operation_limit) (default 10,000) — the worker's far roomier `expr.operation_limit` never enters into it. Measured cost is linear at ~24.15 operations per slice, so the preset documents a ceiling of 400 slices under stock server limits; see -that file's `description` and `internal/product/exprpresetcost_test.go`, -which pins the ceiling as a regression guard. +that file's `description`, [`docs/preset-library.md`](../preset-library.md#transcoding-reference-presets), +and `internal/product/exprpresetcost_test.go`, which pins the ceiling as a +regression guard. ## Current status diff --git a/docs/preset-library.md b/docs/preset-library.md index f897802d..b6bbd960 100644 --- a/docs/preset-library.md +++ b/docs/preset-library.md @@ -198,8 +198,8 @@ The official library also ships five `Transcoding`-category presets under than through a DCC submitter. Each gates on the `attr.worker.tag.ffmpeg = "true"` capability tag, which `sqi-worker` sets automatically on any worker with `ffmpeg` on `PATH` (see [worker capability -tags](worker-capabilities.md)); the two shell-joined segmented variants add an -OS gate on top of that, and the portable one deliberately does not: +tags](worker-capabilities.md)); only the PowerShell-joined segmented variant +adds an OS gate on top of that: - `ffmpeg-transcode` — converts one video file on one worker, start to finish. Base-spec OpenJD (declares no extensions), so it runs on any @@ -211,13 +211,21 @@ OS gate on top of that, and the portable one deliberately does not: scheduler may also reuse one — then join the slices back into a single file. All three need Source Duration entered by hand, since nothing can measure it before submission. Pick one based on your farm: - - `ffmpeg-segment-transcode-bash` — joins with a bash script; needs Linux - or macOS workers. Removes its slice files once the join succeeds. + - `ffmpeg-segment-transcode-bash` — joins with a bash script. Carries no OS + gate: bash is not POSIX-only, since git-bash puts it on Windows too, so + this variant runs on any `ffmpeg=true` worker with `bash` on `PATH`. It + invokes `bash` explicitly with the script as an argument rather than + exec'ing the script directly, because executing a `#!` script is a POSIX + kernel feature Windows has no equivalent of, and the script folds + backslashes in the output path to forward slashes so its `dirname`, + `basename`, and glob work on a Windows path. Removes its slice files once + the join succeeds. - `ffmpeg-segment-transcode-powershell` — joins with a PowerShell script; needs Windows workers. Removes its slice files once the join succeeds. - `ffmpeg-segment-transcode-expr` ("Portable") — needs no shell at all, so - it runs on Linux, macOS, and Windows workers alike; its join file list is - generated by the template itself. That list-building cost is charged at + it runs on any `ffmpeg=true` worker without even requiring `bash`; its join + file list is generated by the template itself. That list-building cost is + charged at submission and grows with the slice count, so it suits jobs of up to 400 slices — past that, use the Bash or PowerShell variant, whose cost does not grow with slice count. Unlike the two shell variants, it leaves its diff --git a/docs/products.md b/docs/products.md index 6aab1c22..90c8e11b 100644 --- a/docs/products.md +++ b/docs/products.md @@ -263,8 +263,9 @@ capability tag, which `sqi-worker` sets automatically on any worker with `ffmpeg` on `PATH` (see [worker capability tags](worker-capabilities.md)); the three segmented variants split a source into slices across the farm and differ in how they join the slices back -together (bash, PowerShell, or a shell-free EXPR template), which OS they can -run on, and whether they delete the slice files afterwards. Full reference: +together (bash, PowerShell, or a shell-free EXPR template), what that join +requires of the worker — only the PowerShell variant is gated to a single OS — +and whether they delete the slice files afterwards. Full reference: [`docs/preset-library.md`](preset-library.md#transcoding-reference-presets). Key points about installed products: diff --git a/presets/sqi/blender-batch-render.yaml b/presets/sqi/blender-batch-render.yaml index e0ff3ffc..9e460616 100644 --- a/presets/sqi/blender-batch-render.yaml +++ b/presets/sqi/blender-batch-render.yaml @@ -5,8 +5,7 @@ description: >- Renders a saved .blend file headless, one frame per task by default; raise the task chunk size to batch a frame range per Blender launch. Requires workers tagged blender=true with blender on PATH. Leave Output Path blank to - use the scene's own output setting, or set it to override. A starting point - - duplicate to customize for your pipeline. + use the scene's own output setting, or set it to override. category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/ffmpeg-segment-transcode-bash.yaml b/presets/sqi/ffmpeg-segment-transcode-bash.yaml index 2bda3173..deed694d 100644 --- a/presets/sqi/ffmpeg-segment-transcode-bash.yaml +++ b/presets/sqi/ffmpeg-segment-transcode-bash.yaml @@ -53,9 +53,11 @@ readme: | ## Requirements `ffmpeg` and `bash` must be on `PATH` for every worker that runs this job, - and workers must be tagged `attr.worker.tag.ffmpeg = true` with - `attr.worker.os.family` in `linux` or `macos`. This is a starting point - - duplicate it to customize for your own pipeline. + and workers must be tagged `attr.worker.tag.ffmpeg = true`. There is + deliberately no `attr.worker.os.family` requirement: what this variant needs + is a working `bash`, not a particular operating system, and tagging the + workers that have one says so directly. This is a starting point - duplicate + it to customize for your own pipeline. category: Transcoding version: 1.0.0 template: @@ -105,8 +107,6 @@ template: attributes: - name: attr.worker.tag.ffmpeg anyOf: ["true"] - - name: attr.worker.os.family - anyOf: ["linux", "macos"] parameterSpace: taskParameterDefinitions: - name: Segment @@ -138,8 +138,6 @@ template: attributes: - name: attr.worker.tag.ffmpeg anyOf: ["true"] - - name: attr.worker.os.family - anyOf: ["linux", "macos"] script: embeddedFiles: - name: join diff --git a/presets/sqi/ffmpeg-sequence-encode.yaml b/presets/sqi/ffmpeg-sequence-encode.yaml index 116bb6b3..afa37261 100644 --- a/presets/sqi/ffmpeg-sequence-encode.yaml +++ b/presets/sqi/ffmpeg-sequence-encode.yaml @@ -2,12 +2,8 @@ name: ffmpeg-sequence-encode title: FFmpeg Sequence Encode description: >- - Turns a rendered image sequence into a movie on one worker - the step that - follows a render job. Point Source Pattern at the sequence using ffmpeg's - printf-style numbering (for example render_%04d.exr); the movie is named - after it, so render_%04d.exr produces render.mp4 in the output directory. - Requires ffmpeg on PATH. A starting point - duplicate to customize for your - pipeline. + Turns a rendered image sequence into a movie on one worker, named after the + sequence's printf-style pattern - render_%04d.exr produces render.mp4. category: Transcoding version: 1.0.0 template: diff --git a/presets/sqi/ffmpeg-transcode.yaml b/presets/sqi/ffmpeg-transcode.yaml index 5ffac8ea..f0d5656d 100644 --- a/presets/sqi/ffmpeg-transcode.yaml +++ b/presets/sqi/ffmpeg-transcode.yaml @@ -2,11 +2,7 @@ name: ffmpeg-transcode title: FFmpeg Transcode description: >- - Converts one video file on one worker, start to finish. Base-spec OpenJD with - no extensions, so it runs on any worker tagged ffmpeg=true whatever that - worker's EXPR limits are. Use a segment variant instead to spread a long - source across the farm. Requires ffmpeg on PATH. A starting point - duplicate - to customize for your pipeline. + Converts one video file on one worker, start to finish. category: Transcoding version: 1.0.0 template: diff --git a/presets/sqi/houdini-rop-render.yaml b/presets/sqi/houdini-rop-render.yaml index 9aa9cdb0..2724d938 100644 --- a/presets/sqi/houdini-rop-render.yaml +++ b/presets/sqi/houdini-rop-render.yaml @@ -3,8 +3,7 @@ name: houdini-rop-render title: Houdini ROP Render description: >- Renders a ROP from a saved hip file with hython, in frame chunks. Requires - workers tagged houdini=true with hython on PATH. A starting point - - duplicate to customize for your pipeline. + workers tagged houdini=true with hython on PATH. category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/maya-layer-render.yaml b/presets/sqi/maya-layer-render.yaml index 6f12d38e..56bb67f1 100644 --- a/presets/sqi/maya-layer-render.yaml +++ b/presets/sqi/maya-layer-render.yaml @@ -5,8 +5,7 @@ description: >- Renders one render layer of a saved Maya scene with the Render batch command, one frame per task by default; raise the task chunk size to batch multiple frames per Render launch. Requires workers tagged maya=true with - Render on PATH. For all renderable layers use maya-scene-render. A starting - point - duplicate to customize for your pipeline. + Render on PATH. For all renderable layers use maya-scene-render. category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/maya-scene-render.yaml b/presets/sqi/maya-scene-render.yaml index c41f2ffd..b2e7c2df 100644 --- a/presets/sqi/maya-scene-render.yaml +++ b/presets/sqi/maya-scene-render.yaml @@ -6,8 +6,7 @@ description: >- settings mark renderable), one frame per task by default; raise the task chunk size to batch multiple frames per Render launch. Requires workers tagged maya=true with Render on PATH. For a single render layer use - maya-layer-render. A starting point - duplicate to customize for your - pipeline. + maya-layer-render. category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/mistika-boutique-render.yaml b/presets/sqi/mistika-boutique-render.yaml index 48074c60..21e19d63 100644 --- a/presets/sqi/mistika-boutique-render.yaml +++ b/presets/sqi/mistika-boutique-render.yaml @@ -4,8 +4,7 @@ title: Mistika Ultima & Boutique Render description: >- Renders a saved Mistika .rnd job with the mistika binary, in frame chunks. Requires workers tagged mistika=true with mistika on PATH. For a single movie - file, submit Frames "1". A starting point - duplicate to customize for your - pipeline. + file, submit Frames "1". category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/mistika-vr-render.yaml b/presets/sqi/mistika-vr-render.yaml index 89718b34..42814332 100644 --- a/presets/sqi/mistika-vr-render.yaml +++ b/presets/sqi/mistika-vr-render.yaml @@ -4,8 +4,7 @@ title: Mistika VR Render description: >- Renders a saved Mistika VR .rnd job with the vr binary, in frame chunks. Requires workers tagged mistikavr=true with vr on PATH. For a single movie - file, submit Frames "1". A starting point - duplicate to customize for your - pipeline. + file, submit Frames "1". category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/mistika-workflows-render.yaml b/presets/sqi/mistika-workflows-render.yaml index ec85c6b1..d28fbd6c 100644 --- a/presets/sqi/mistika-workflows-render.yaml +++ b/presets/sqi/mistika-workflows-render.yaml @@ -4,8 +4,7 @@ title: Mistika Workflows Render description: >- Renders a saved Mistika Workflows .rnd job with the workflows binary, in frame chunks. Requires workers tagged mistikaworkflows=true with workflows on PATH. - For a single movie file, submit Frames "1". A starting point - duplicate to - customize for your pipeline. + For a single movie file, submit Frames "1". category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/nuke-script-render.yaml b/presets/sqi/nuke-script-render.yaml index 07762dea..daa6fe7c 100644 --- a/presets/sqi/nuke-script-render.yaml +++ b/presets/sqi/nuke-script-render.yaml @@ -4,8 +4,7 @@ title: Nuke Script Render description: >- Renders all enabled Write nodes of a saved Nuke script in frame chunks. Requires workers tagged nuke=true with nuke on PATH. For a single Write node - use nuke-write-render. A starting point - duplicate to customize for your - pipeline. + use nuke-write-render. category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/nuke-write-render.yaml b/presets/sqi/nuke-write-render.yaml index f0d9aa0a..ff9f38ed 100644 --- a/presets/sqi/nuke-write-render.yaml +++ b/presets/sqi/nuke-write-render.yaml @@ -3,8 +3,7 @@ name: nuke-write-render title: Nuke Write Render description: >- Renders one Write node from a saved Nuke script in frame chunks. Requires - workers tagged nuke=true with nuke on PATH. A starting point - duplicate - to customize for your pipeline. + workers tagged nuke=true with nuke on PATH. category: Rendering version: 1.0.0 template: diff --git a/test/integration/ffmpeg_presets_test.go b/test/integration/ffmpeg_presets_test.go index 0973e8ac..51917d9f 100644 --- a/test/integration/ffmpeg_presets_test.go +++ b/test/integration/ffmpeg_presets_test.go @@ -433,18 +433,29 @@ func TestFFmpegPreset_PortableSegmentTranscodeJoins(t *testing.T) { } // TestFFmpegPreset_BashSegmentTranscodeJoins runs the bash-joined variant, -// whose template gates on attr.worker.os.family anyOf [linux, macos]. +// whose template gates on attr.worker.tag.ffmpeg ALONE — it carries no +// attr.worker.os.family requirement, because bash is not a POSIX-only +// interpreter: git-bash puts it on Windows too. The skip below therefore keys +// on bash being on PATH rather than on GOOS, which is the preset's actual +// requirement restated. // // It runs on darwin as well as linux, and that is load-bearing rather than // incidental: until scheduler.osFamily landed, a Mac worker reported GOOS -// "darwin" against a requirement that can only legally say "macos", so this +// "darwin" against a requirement that could only legally say "macos", so this // preset validated, submitted, and then waited forever for a worker that could // not exist. An earlier revision of this test skipped darwin FOR THAT REASON. -// Running here is the end-to-end proof that a Mac can now take the work — a +// Running here is the end-to-end proof that a Mac can take the work — a // regression would show up as this test timing out rather than failing fast. +// +// The template invokes "command: bash" with the script as an argument rather +// than exec'ing the shebang script directly, since direct shebang execution is +// a POSIX kernel feature (binfmt_script) Windows has no equivalent of. On a +// Windows host this case is the only automated coverage of that invocation and +// of the script's backslash folding; it has NOT been run on a real Windows +// host, so treat a green run here as covering the POSIX half only. func TestFFmpegPreset_BashSegmentTranscodeJoins(t *testing.T) { - if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { - t.Skipf("ffmpeg-segment-transcode-bash requires a linux or macos worker; GOOS=%s", runtime.GOOS) + if _, err := exec.LookPath("bash"); err != nil { + t.Skipf("ffmpeg-segment-transcode-bash requires bash on PATH: %v", err) } runSegmentPreset(t, "ffmpeg-segment-transcode-bash", false) } From fa26dc9b8327f401b30fc5d10b3134ac1474aed8 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Tue, 18 Aug 2026 21:40:34 -0700 Subject: [PATCH 07/10] docs(products): standardize descriptions and add readmes to every shipped product --- internal/product/builtins/container.yaml | 2 +- internal/product/builtins/python.yaml | 2 +- internal/product/builtins/script.yaml | 2 +- internal/product/readmequality_test.go | 184 ++++++++++++++++++ presets/sqi/blender-batch-render.yaml | 37 +++- .../sqi/ffmpeg-segment-transcode-bash.yaml | 28 ++- .../sqi/ffmpeg-segment-transcode-expr.yaml | 26 ++- .../ffmpeg-segment-transcode-powershell.yaml | 30 ++- presets/sqi/ffmpeg-sequence-encode.yaml | 35 +++- presets/sqi/ffmpeg-transcode.yaml | 34 +++- presets/sqi/houdini-rop-render.yaml | 32 ++- presets/sqi/maya-layer-render.yaml | 43 +++- presets/sqi/maya-scene-render.yaml | 42 +++- presets/sqi/mistika-boutique-render.yaml | 32 ++- presets/sqi/mistika-vr-render.yaml | 31 ++- presets/sqi/mistika-workflows-render.yaml | 32 ++- presets/sqi/nuke-script-render.yaml | 33 +++- presets/sqi/nuke-write-render.yaml | 29 ++- presets/testing/test-render-bash.yaml | 45 ++++- presets/testing/test-render-powershell.yaml | 45 ++++- presets/testing/test-steps-bash.yaml | 44 ++++- presets/testing/test-steps-powershell.yaml | 45 ++++- 22 files changed, 735 insertions(+), 98 deletions(-) create mode 100644 internal/product/readmequality_test.go diff --git a/internal/product/builtins/container.yaml b/internal/product/builtins/container.yaml index dd912b48..60eacaa6 100644 --- a/internal/product/builtins/container.yaml +++ b/internal/product/builtins/container.yaml @@ -20,7 +20,7 @@ readme: | docker run --rm registry.example.com/team/nuke:15.1 ``` - ## Notes + ## Requirements The step declares `attr.worker.tag.docker`, so it will only ever be scheduled onto workers tagged as having Docker. A worker without that tag is not a diff --git a/internal/product/builtins/python.yaml b/internal/product/builtins/python.yaml index 5db30362..239bc809 100644 --- a/internal/product/builtins/python.yaml +++ b/internal/product/builtins/python.yaml @@ -22,7 +22,7 @@ readme: | print("running on", sys.platform) ``` - ## Notes + ## Requirements The script is delivered as an OpenJD embedded file, so it reaches the worker as a real `script.py` on disk rather than as a command-line argument. That is why diff --git a/internal/product/builtins/script.yaml b/internal/product/builtins/script.yaml index 0764614c..fbe45442 100644 --- a/internal/product/builtins/script.yaml +++ b/internal/product/builtins/script.yaml @@ -20,7 +20,7 @@ readme: | ffmpeg -i input.mov -c:v libx264 output.mp4 ``` - ## Notes + ## Requirements The command runs on whichever worker picks up the task, as the account that worker runs under, in a session directory the worker creates. Nothing is diff --git a/internal/product/readmequality_test.go b/internal/product/readmequality_test.go new file mode 100644 index 00000000..6e3a6bc0 --- /dev/null +++ b/internal/product/readmequality_test.go @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package product + +import ( + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + "github.com/uberware/sqi/internal/fsutil" +) + +// shippedDefinitionDirs is every directory whose products this repo authors and +// ships: the embedded built-ins plus both preset trees. All three reach users +// through the same renderer, so all three answer to the same rules. +var shippedDefinitionDirs = []string{ + "builtins", + "../../presets/sqi", + "../../presets/testing", +} + +// shippedDefinition is one product definition file, decoded far enough to check +// its prose. The template is deliberately not decoded -- the schema tests own +// that. +type shippedDefinition struct { + Name string `yaml:"name"` + Title string `yaml:"title"` + Description string `yaml:"description"` + Readme string `yaml:"readme"` +} + +// loadShippedDefinitions reads every product definition this repo ships. +func loadShippedDefinitions(t *testing.T) map[string]shippedDefinition { + t.Helper() + out := map[string]shippedDefinition{} + for _, dir := range shippedDefinitionDirs { + files, err := fsutil.Glob(filepath.Join(dir, "*.yaml")) + if err != nil { + t.Fatalf("glob %s: %v", dir, err) + } + if len(files) == 0 { + t.Fatalf("no definitions found under %s", dir) + } + for _, f := range files { + data, readErr := os.ReadFile(f) + if readErr != nil { + t.Fatalf("read %s: %v", f, readErr) + } + var def shippedDefinition + if err := yaml.Unmarshal(data, &def); err != nil { + t.Fatalf("parse %s: %v", f, err) + } + out[f] = def + } + } + return out +} + +// Every shipped product carries a readme. These are the products users meet +// first, and the readme is where a product explains what it produces -- a +// product without one leaves its detail page showing nothing but a template. +func TestShippedDefinitions_AllHaveAReadme(t *testing.T) { + t.Parallel() + for path, def := range loadShippedDefinitions(t) { + if strings.TrimSpace(def.Readme) == "" { + t.Errorf("%s: no readme", path) + } + } +} + +// Descriptions are the short, plain-text blurb: rendered into an unclamped +// picker card and a native Blender EnumProperty tooltip, and the only field +// product search matches. Markdown in one renders as literal punctuation. +func TestShippedDefinitions_DescriptionsAreShortPlainText(t *testing.T) { + t.Parallel() + // 3 sentences is the house standard; the 500-rune cap in limits.go is the + // hard bound, and this is the editorial one well inside it. + const maxSentences = 3 + for path, def := range loadShippedDefinitions(t) { + desc := strings.TrimSpace(def.Description) + if desc == "" { + t.Errorf("%s: no description", path) + continue + } + if n := utf8.RuneCountInString(desc); n > MaxDescriptionLen { + t.Errorf("%s: description is %d characters, limit is %d", path, n, MaxDescriptionLen) + } + if n := strings.Count(desc, ".") + strings.Count(desc, "!") + strings.Count(desc, "?"); n > maxSentences { + t.Errorf("%s: description reads as %d sentences, house standard is at most %d", + path, n, maxSentences) + } + for _, m := range []struct{ name, marker string }{ + {"a heading", "#"}, + {"bold or a list bullet", "**"}, + {"inline code", "`"}, + {"a link", "]("}, + } { + if strings.Contains(desc, m.marker) { + t.Errorf("%s: description contains %s (%q) -- descriptions are plain text", + path, m.name, m.marker) + } + } + } +} + +// Readmes may use only the subset web/src/components/Markdown.tsx implements. +// Anything else renders as literal text in the UI -- silently, with no error, +// which is exactly how three shipped presets once shipped nested lists that +// rendered as stray paragraphs. +func TestShippedDefinitions_ReadmesUseTheSupportedSubset(t *testing.T) { + t.Parallel() + // Substring matching is too crude for these: `ffmpeg -ss -t ...` + // in prose contains both "<" and "> ". Block constructs are therefore + // matched at line starts, and the HTML check names tags an author might + // genuinely expect to render. Raw HTML is harmless either way -- the + // renderer emits React elements, so it escapes -- the check is about a + // mistaken expectation, not a vulnerability. + inlineUnsupported := []struct{ name, marker string }{ + {"an image", "!["}, + } + htmlTags := []string{"

", "", "", "", "", " MaxReadmeLen { + t.Errorf("%s: readme is %d characters, limit is %d", path, n, MaxReadmeLen) + } + for _, u := range inlineUnsupported { + if strings.Contains(readme, u.marker) { + t.Errorf("%s: readme contains %s (%q), which the renderer does not support "+ + "and will render as literal text", path, u.name, u.marker) + } + } + for _, tag := range htmlTags { + if strings.Contains(strings.ToLower(readme), tag) { + t.Errorf("%s: readme contains raw HTML (%q), which renders as literal text", path, tag) + } + } + for i, line := range strings.Split(readme, "\n") { + switch { + case strings.HasPrefix(line, "> "), line == ">": + t.Errorf("%s:%d: blockquote -- not supported, renders as literal text", path, i+1) + case strings.HasPrefix(line, "|"): + t.Errorf("%s:%d: table row -- not supported, renders as literal text", path, i+1) + } + } + // The renderer's list regexes are anchored at column 0, so an indented + // list item matches nothing and falls through into a paragraph with a + // stray leading marker. The structure is lost with no error anywhere. + for i, line := range strings.Split(readme, "\n") { + trimmed := strings.TrimLeft(line, " \t") + if len(line) == len(trimmed) { + continue + } + if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") { + t.Errorf("%s:%d: indented list item %q -- nesting is not supported", + path, i+1, strings.TrimSpace(line)) + } + } + } +} + +// Every readme opens with a level-1 heading. The renderer offsets it to

, +// nesting it under the detail page's own

/

, so this is what gives the +// rendered readme a title rather than starting mid-prose. +func TestShippedDefinitions_ReadmesOpenWithAHeading(t *testing.T) { + t.Parallel() + for path, def := range loadShippedDefinitions(t) { + readme := strings.TrimSpace(def.Readme) + if readme == "" { + continue // reported by TestShippedDefinitions_AllHaveAReadme + } + first, _, _ := strings.Cut(readme, "\n") + if !strings.HasPrefix(first, "# ") { + t.Errorf("%s: readme opens with %q, want a `# ` heading", path, first) + } + } +} diff --git a/presets/sqi/blender-batch-render.yaml b/presets/sqi/blender-batch-render.yaml index 9e460616..d7ff4093 100644 --- a/presets/sqi/blender-batch-render.yaml +++ b/presets/sqi/blender-batch-render.yaml @@ -2,10 +2,39 @@ name: blender-batch-render title: Blender Batch Render description: >- - Renders a saved .blend file headless, one frame per task by default; raise - the task chunk size to batch a frame range per Blender launch. Requires - workers tagged blender=true with blender on PATH. Leave Output Path blank to - use the scene's own output setting, or set it to override. + Renders a saved .blend file headless with Blender's batch mode, one frame + per task by default. +readme: | + # Blender Batch Render + + Renders a saved `.blend` file with `blender -b`, writing image files to the + output path you choose. Each task renders a contiguous slice of the frame + range, one frame per task by default. + + ## Parameters + + - **Scene File** - the `.blend` file to render. It must be reachable from every + worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + - **Output Path** - the output path passed to Blender as `-o`. Blender treats + it as a prefix and appends the frame number and extension. Left empty, the + path saved in the blend file is used. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking. The + `SQI_CHUNK_BOUNDS` extension gives each task the first and last frame of its + own slice, which is what lets the command pass a start and an end rather than + a frame list. + + One frame per task is the default. Raise the chunk size to batch a frame range + into a single Blender launch, which trades parallelism for fewer scene loads. + + ## Requirements + + - Workers tagged `attr.worker.tag.blender = true` + - `blender` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/ffmpeg-segment-transcode-bash.yaml b/presets/sqi/ffmpeg-segment-transcode-bash.yaml index deed694d..c8aa847e 100644 --- a/presets/sqi/ffmpeg-segment-transcode-bash.yaml +++ b/presets/sqi/ffmpeg-segment-transcode-bash.yaml @@ -13,20 +13,6 @@ readme: | output file. The join step is a bash script, so this variant needs Linux or macOS workers - use the Portable variant on Windows or a mixed farm. - ## Steps - - 1. **Transcode** - fans out into one task per slice. Each task runs: - `ffmpeg -ss -t -i SourceFile ...` and writes - `_seg_<00001>.` beside `OutputFile`. - The number of tasks is `ceil(DurationSeconds / SegmentSeconds)`. - 2. **Join** - depends on every Transcode task, and runs an embedded bash - script on the worker. The script globs `/_seg_*.` next - to `OutputFile` - zero-padded slice numbers make lexical glob order the - same as numeric order - builds a concat file list from the matches, - then runs `ffmpeg -f concat -safe 0` to copy the streams into - `OutputFile` with no re-encode. On success it deletes the slice files - it just joined. - ## Parameters - **SourceFile** - the video to split. @@ -44,7 +30,19 @@ readme: | - **VideoCodec**, **Quality** (CRF, lower is better), **AudioCodec** - passed straight to `ffmpeg` for every slice. - ## Cleanup + ## How work is divided + + 1. **Transcode** - fans out into one task per slice. Each task runs: + `ffmpeg -ss -t -i SourceFile ...` and writes + `_seg_<00001>.` beside `OutputFile`. + The number of tasks is `ceil(DurationSeconds / SegmentSeconds)`. + 2. **Join** - depends on every Transcode task, and runs an embedded bash + script on the worker. The script globs `/_seg_*.` next + to `OutputFile` - zero-padded slice numbers make lexical glob order the + same as numeric order - builds a concat file list from the matches, + then runs `ffmpeg -f concat -safe 0` to copy the streams into + `OutputFile` with no re-encode. On success it deletes the slice files + it just joined. The join script removes every slice file it globbed once the concat succeeds. If the join fails, the slice files are left on disk for you to diff --git a/presets/sqi/ffmpeg-segment-transcode-expr.yaml b/presets/sqi/ffmpeg-segment-transcode-expr.yaml index e1d40352..4df08fe0 100644 --- a/presets/sqi/ffmpeg-segment-transcode-expr.yaml +++ b/presets/sqi/ffmpeg-segment-transcode-expr.yaml @@ -15,18 +15,6 @@ readme: | extension, so the same template runs unchanged on Linux, macOS and Windows workers. - ## Steps - - 1. **Transcode** - fans out into one task per slice. Each task runs: - `ffmpeg -ss -t -i SourceFile ...` and writes - `_seg_<00001>.` beside `OutputFile`. - The number of tasks is `ceil(DurationSeconds / SegmentSeconds)`. - 2. **Join** - depends on every Transcode task. An embedded file lists every - slice path as `file ''`, one per line, built with an `EXPR` - `join()` expression over a `range()` comprehension - no script, no loop - on the worker. `ffmpeg -f concat -safe 0` reads that list and copies the - streams into `OutputFile` with no re-encode. - ## Parameters - **SourceFile** - the video to split. @@ -44,7 +32,17 @@ readme: | - **VideoCodec**, **Quality** (CRF, lower is better), **AudioCodec** - passed straight to `ffmpeg` for every slice. - ## Scaling + ## How work is divided + + 1. **Transcode** - fans out into one task per slice. Each task runs: + `ffmpeg -ss -t -i SourceFile ...` and writes + `_seg_<00001>.` beside `OutputFile`. + The number of tasks is `ceil(DurationSeconds / SegmentSeconds)`. + 2. **Join** - depends on every Transcode task. An embedded file lists every + slice path as `file ''`, one per line, built with an `EXPR` + `join()` expression over a `range()` comprehension - no script, no loop + on the worker. `ffmpeg -f concat -safe 0` reads that list and copies the + streams into `OutputFile` with no re-encode. Building the join step's file list is charged against this template's `EXPR` evaluation budget at submission time, and that cost grows with the @@ -54,8 +52,6 @@ readme: | regardless of slice count (it lists slices with a directory glob at run time, not at submission time). - ## Cleanup - Unlike the Bash and PowerShell variants, this template does not remove the per-slice files after a successful join - there is no shell step to do it in. They are left beside `OutputFile` for you to delete. diff --git a/presets/sqi/ffmpeg-segment-transcode-powershell.yaml b/presets/sqi/ffmpeg-segment-transcode-powershell.yaml index da37f30f..7afbe5ac 100644 --- a/presets/sqi/ffmpeg-segment-transcode-powershell.yaml +++ b/presets/sqi/ffmpeg-segment-transcode-powershell.yaml @@ -13,21 +13,6 @@ readme: | output file. The join step is a PowerShell script, so this variant needs Windows workers - use the Portable variant on a mixed farm. - ## Steps - - 1. **Transcode** - fans out into one task per slice. Each task runs: - `ffmpeg -ss -t -i SourceFile ...` and writes - `_seg_<00001>.` beside `OutputFile`. - The number of tasks is `ceil(DurationSeconds / SegmentSeconds)`. - 2. **Join** - depends on every Transcode task, and runs an embedded - PowerShell script on the worker. The script lists - `\_seg_*` next to `OutputFile` with `Get-ChildItem`, - sorted by name - zero-padded slice numbers make that the same as - numeric order - writes the matches to a BOM-less UTF-8 concat file list - (a BOM breaks the concat demuxer's parse of the first line), then runs - `ffmpeg -f concat -safe 0` to copy the streams into `OutputFile` with no - re-encode. On success it deletes the slice files it just joined. - ## Parameters - **SourceFile** - the video to split. @@ -45,7 +30,20 @@ readme: | - **VideoCodec**, **Quality** (CRF, lower is better), **AudioCodec** - passed straight to `ffmpeg` for every slice. - ## Cleanup + ## How work is divided + + 1. **Transcode** - fans out into one task per slice. Each task runs: + `ffmpeg -ss -t -i SourceFile ...` and writes + `_seg_<00001>.` beside `OutputFile`. + The number of tasks is `ceil(DurationSeconds / SegmentSeconds)`. + 2. **Join** - depends on every Transcode task, and runs an embedded + PowerShell script on the worker. The script lists + `\_seg_*` next to `OutputFile` with `Get-ChildItem`, + sorted by name - zero-padded slice numbers make that the same as + numeric order - writes the matches to a BOM-less UTF-8 concat file list + (a BOM breaks the concat demuxer's parse of the first line), then runs + `ffmpeg -f concat -safe 0` to copy the streams into `OutputFile` with no + re-encode. On success it deletes the slice files it just joined. The join script removes every slice file it listed once the concat succeeds. If the join fails, the slice files are left on disk for you to diff --git a/presets/sqi/ffmpeg-sequence-encode.yaml b/presets/sqi/ffmpeg-sequence-encode.yaml index afa37261..8b8a9fb5 100644 --- a/presets/sqi/ffmpeg-sequence-encode.yaml +++ b/presets/sqi/ffmpeg-sequence-encode.yaml @@ -2,8 +2,39 @@ name: ffmpeg-sequence-encode title: FFmpeg Sequence Encode description: >- - Turns a rendered image sequence into a movie on one worker, named after the - sequence's printf-style pattern - render_%04d.exr produces render.mp4. + Encodes a numbered image sequence into a single video file on one worker. +readme: | + # FFmpeg Sequence Encode + + Encodes a numbered image sequence - the frames a render produces - into one + video file. A single `ffmpeg` invocation on one worker reads the sequence and + writes the movie. + + ## Parameters + + - **Source Pattern** - the image sequence to read, as an ffmpeg pattern such as + `render.%04d.exr`. It must be reachable from the worker that runs the job. + - **Start Frame** - the first frame number to read, passed as `-start_number`. + - **End Frame** - the last frame number to read. Together with the start frame + it decides how many frames are encoded. + - **Frame Rate** - frames per second of the resulting video. + - **Output Directory** - where the encoded video is written. + - **Video Codec** - `libx264` or `libx265`. + - **Quality** - the constant rate factor passed as `-crf`. Lower is better + quality and a larger file. + + ## How work is divided + + It is not. This product defines one step with one task: encoding a sequence + into a single container is inherently sequential. + + The frame count handed to ffmpeg is computed from the start and end frames by + an expression, which is why the template declares the `EXPR` extension. + + ## Requirements + + - Workers tagged `attr.worker.tag.ffmpeg = true` + - `ffmpeg` on the worker's `PATH` category: Transcoding version: 1.0.0 template: diff --git a/presets/sqi/ffmpeg-transcode.yaml b/presets/sqi/ffmpeg-transcode.yaml index f0d5656d..97a489bd 100644 --- a/presets/sqi/ffmpeg-transcode.yaml +++ b/presets/sqi/ffmpeg-transcode.yaml @@ -2,7 +2,39 @@ name: ffmpeg-transcode title: FFmpeg Transcode description: >- - Converts one video file on one worker, start to finish. + Converts one video file to another format on a single worker, start to + finish. +readme: | + # FFmpeg Transcode + + Transcodes one video file into another with a single `ffmpeg` invocation on one + worker. The whole file is converted by one task, so the job finishes no faster + than one machine can encode it. + + Use FFmpeg Segment Transcode instead to split a long source across the farm. + + ## Parameters + + - **Source File** - the video to convert. It must be reachable from the worker + that runs the job. + - **Output File** - where the converted video is written. Its extension decides + the container format. + - **Video Codec** - `libx264`, `libx265`, or `copy` to pass the video through + without re-encoding. + - **Quality** - the constant rate factor passed as `-crf`. Lower is better + quality and a larger file; 20 is a reasonable default. + - **Audio Codec** - `aac`, or `copy` to pass the audio through untouched. + + ## How work is divided + + It is not. This product defines one step with one task and declares no OpenJD + extensions, which also means it is never withheld from a worker on expression + limits. + + ## Requirements + + - Workers tagged `attr.worker.tag.ffmpeg = true` + - `ffmpeg` on the worker's `PATH` category: Transcoding version: 1.0.0 template: diff --git a/presets/sqi/houdini-rop-render.yaml b/presets/sqi/houdini-rop-render.yaml index 2724d938..47250bf1 100644 --- a/presets/sqi/houdini-rop-render.yaml +++ b/presets/sqi/houdini-rop-render.yaml @@ -2,8 +2,36 @@ name: houdini-rop-render title: Houdini ROP Render description: >- - Renders a ROP from a saved hip file with hython, in frame chunks. Requires - workers tagged houdini=true with hython on PATH. + Renders a ROP from a saved Houdini hip file with hython, in frame chunks. +readme: | + # Houdini ROP Render + + Renders one render operator (ROP) from a saved Houdini `.hip` file. Each task + runs `hython` against a short Python script the template carries as an embedded + file, which opens the hip, finds the ROP and renders the frames it was given. + + Each task renders a contiguous chunk of frames, ten per task by default. + + ## Parameters + + - **Scene File** - the `.hip` file to render. It must be reachable from every + worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + - **ROP Path** - the full node path of the ROP to render, for example + `/out/mantra1`. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking, ten frames per + task by default. Each task receives its own frames as a list and passes them + straight to the embedded script, so this preset needs no chunk-bounds + extension. + + ## Requirements + + - Workers tagged `attr.worker.tag.houdini = true` + - `hython` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/maya-layer-render.yaml b/presets/sqi/maya-layer-render.yaml index 56bb67f1..45ff10cf 100644 --- a/presets/sqi/maya-layer-render.yaml +++ b/presets/sqi/maya-layer-render.yaml @@ -2,10 +2,45 @@ name: maya-layer-render title: Maya Layer Render description: >- - Renders one render layer of a saved Maya scene with the Render batch - command, one frame per task by default; raise the task chunk size to batch - multiple frames per Render launch. Requires workers tagged maya=true with - Render on PATH. For all renderable layers use maya-scene-render. + Renders one named render layer of a saved Maya scene with the Render batch + command, one frame per task by default. +readme: | + # Maya Layer Render + + Renders a single named layer of a saved Maya scene with Maya's `Render` batch + command, writing image files to a directory you choose. Use Maya Scene Render + instead to render every layer the scene marks renderable. + + Each task renders a contiguous slice of the frame range, one frame per task by + default. + + ## Parameters + + - **Scene File** - the `.ma` or `.mb` file to render. It must be reachable from + every worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + - **Output Directory** - where rendered images are written, passed to Maya as + `-rd`. + - **Renderer** - the Maya renderer token passed as `-r`. `file` uses the + renderer stored in the scene. + - **Render Layer** - the layer to render, passed as `-rl`. Defaults to + `masterLayer`. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking. The + `SQI_CHUNK_BOUNDS` extension gives each task the first and last frame of its + own slice, which is what lets the command pass a start and an end rather than + a frame list. + + One frame per task is the default. Raise the chunk size to batch several frames + into a single Maya launch, which trades parallelism for fewer scene loads. + + ## Requirements + + - Workers tagged `attr.worker.tag.maya = true` + - `Render` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/maya-scene-render.yaml b/presets/sqi/maya-scene-render.yaml index b2e7c2df..5fbd34e1 100644 --- a/presets/sqi/maya-scene-render.yaml +++ b/presets/sqi/maya-scene-render.yaml @@ -2,11 +2,43 @@ name: maya-scene-render title: Maya Scene Render description: >- - Renders a saved Maya scene's renderable layers (whatever the scene's render - settings mark renderable), one frame per task by default; raise the task - chunk size to batch multiple frames per Render launch. Requires workers - tagged maya=true with Render on PATH. For a single render layer use - maya-layer-render. + Renders every renderable layer of a saved Maya scene with the Render batch + command, one frame per task by default. +readme: | + # Maya Scene Render + + Renders a saved Maya scene with Maya's `Render` batch command, writing image + files to a directory you choose. Every layer the scene marks renderable is + rendered; use Maya Layer Render instead to render one named layer. + + Each task renders a contiguous slice of the frame range, one frame per task by + default. + + ## Parameters + + - **Scene File** - the `.ma` or `.mb` file to render. It must be reachable from + every worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + - **Output Directory** - where rendered images are written, passed to Maya as + `-rd`. + - **Renderer** - the Maya renderer token passed as `-r`. `file` uses the + renderer stored in the scene. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking, one frame per + task by default. Raise the chunk size to batch several frames into a single + Maya launch, which trades parallelism for fewer scene loads. + + The `SQI_CHUNK_BOUNDS` extension gives each task the first and last frame of + its own slice, which is what lets the command pass `-s` and `-e` rather than a + frame list. + + ## Requirements + + - Workers tagged `attr.worker.tag.maya = true` + - `Render` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/mistika-boutique-render.yaml b/presets/sqi/mistika-boutique-render.yaml index 21e19d63..c60e6691 100644 --- a/presets/sqi/mistika-boutique-render.yaml +++ b/presets/sqi/mistika-boutique-render.yaml @@ -2,9 +2,35 @@ name: mistika-boutique-render title: Mistika Ultima & Boutique Render description: >- - Renders a saved Mistika .rnd job with the mistika binary, in frame chunks. - Requires workers tagged mistika=true with mistika on PATH. For a single movie - file, submit Frames "1". + Renders a saved Mistika Boutique project on the command line, in frame + chunks. +readme: | + # Mistika Boutique Render + + Renders a saved Mistika Boutique project by invoking `mistika` in render mode. + Each task renders a contiguous slice of the frame range. + + ## Parameters + + - **Scene File** - the Mistika project to render. It must be reachable from + every worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking, ten frames per + task by default. The `SQI_CHUNK_BOUNDS` extension gives each task the first and + last frame of its own slice, which is what lets the command pass `-s` and `-e` + rather than a frame list. + + There is no in-application submitter for this product: build the job from the + sqi web UI or the Python client rather than from inside the application. + + ## Requirements + + - Workers tagged `attr.worker.tag.mistika = true` + - `mistika` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/mistika-vr-render.yaml b/presets/sqi/mistika-vr-render.yaml index 42814332..34bc08aa 100644 --- a/presets/sqi/mistika-vr-render.yaml +++ b/presets/sqi/mistika-vr-render.yaml @@ -2,9 +2,34 @@ name: mistika-vr-render title: Mistika VR Render description: >- - Renders a saved Mistika VR .rnd job with the vr binary, in frame chunks. - Requires workers tagged mistikavr=true with vr on PATH. For a single movie - file, submit Frames "1". + Renders a saved Mistika VR project on the command line, in frame chunks. +readme: | + # Mistika VR Render + + Renders a saved Mistika VR project by invoking `vr` in render mode. Each task + renders a contiguous slice of the frame range. + + ## Parameters + + - **Scene File** - the Mistika VR project to render. It must be reachable from + every worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking, ten frames per + task by default. The `SQI_CHUNK_BOUNDS` extension gives each task the first and + last frame of its own slice, which is what lets the command pass `-s` and `-e` + rather than a frame list. + + There is no in-application submitter for this product: build the job from the + sqi web UI or the Python client rather than from inside the application. + + ## Requirements + + - Workers tagged `attr.worker.tag.mistikavr = true` + - `vr` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/mistika-workflows-render.yaml b/presets/sqi/mistika-workflows-render.yaml index d28fbd6c..cf74ad6c 100644 --- a/presets/sqi/mistika-workflows-render.yaml +++ b/presets/sqi/mistika-workflows-render.yaml @@ -2,9 +2,35 @@ name: mistika-workflows-render title: Mistika Workflows Render description: >- - Renders a saved Mistika Workflows .rnd job with the workflows binary, in frame - chunks. Requires workers tagged mistikaworkflows=true with workflows on PATH. - For a single movie file, submit Frames "1". + Renders a saved Mistika Workflows graph on the command line, in frame + chunks. +readme: | + # Mistika Workflows Render + + Renders a saved Mistika Workflows graph by invoking `workflows` in render mode. + Each task renders a contiguous slice of the frame range. + + ## Parameters + + - **Scene File** - the Workflows graph to render. It must be reachable from + every worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking, ten frames per + task by default. The `SQI_CHUNK_BOUNDS` extension gives each task the first and + last frame of its own slice, which is what lets the command pass `-s` and `-e` + rather than a frame list. + + There is no in-application submitter for this product: build the job from the + sqi web UI or the Python client rather than from inside the application. + + ## Requirements + + - Workers tagged `attr.worker.tag.mistikaworkflows = true` + - `workflows` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/nuke-script-render.yaml b/presets/sqi/nuke-script-render.yaml index daa6fe7c..5ea0bc80 100644 --- a/presets/sqi/nuke-script-render.yaml +++ b/presets/sqi/nuke-script-render.yaml @@ -2,9 +2,36 @@ name: nuke-script-render title: Nuke Script Render description: >- - Renders all enabled Write nodes of a saved Nuke script in frame chunks. - Requires workers tagged nuke=true with nuke on PATH. For a single Write node - use nuke-write-render. + Renders every enabled Write node of a saved Nuke script, in frame chunks. +readme: | + # Nuke Script Render + + Renders a saved Nuke script with `nuke -x`, executing every Write node the + script leaves enabled. Use Nuke Write Render instead to render one named Write + node. + + Each task renders a contiguous chunk of frames, ten per task by default. + + ## Parameters + + - **Scene File** - the `.nk` script to render. It must be reachable from every + worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking, ten frames per + task by default. Each task receives its own frames as a list and passes them to + Nuke's `-F` flag directly, so this preset needs no chunk-bounds extension. + + Raise or lower the chunk size to trade parallelism against the cost of loading + the script once per task. + + ## Requirements + + - Workers tagged `attr.worker.tag.nuke = true` + - `nuke` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/sqi/nuke-write-render.yaml b/presets/sqi/nuke-write-render.yaml index ff9f38ed..49d9818b 100644 --- a/presets/sqi/nuke-write-render.yaml +++ b/presets/sqi/nuke-write-render.yaml @@ -2,8 +2,33 @@ name: nuke-write-render title: Nuke Write Render description: >- - Renders one Write node from a saved Nuke script in frame chunks. Requires - workers tagged nuke=true with nuke on PATH. + Renders one named Write node from a saved Nuke script, in frame chunks. +readme: | + # Nuke Write Render + + Renders a single named Write node from a saved Nuke script with `nuke -x -X`. + Use Nuke Script Render instead to render every enabled Write node in one job. + + Each task renders a contiguous chunk of frames, ten per task by default. + + ## Parameters + + - **Scene File** - the `.nk` script to render. It must be reachable from every + worker that runs the job. + - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or + `1-100:2` for every second frame. + - **Write Node** - the name of the Write node to render, passed to Nuke as `-X`. + + ## How work is divided + + The frame range expands into tasks through OpenJD task chunking, ten frames per + task by default. Each task receives its own frames as a list and passes them to + Nuke's `-F` flag directly, so this preset needs no chunk-bounds extension. + + ## Requirements + + - Workers tagged `attr.worker.tag.nuke = true` + - `nuke` on the worker's `PATH` category: Rendering version: 1.0.0 template: diff --git a/presets/testing/test-render-bash.yaml b/presets/testing/test-render-bash.yaml index 01cb244f..5fc59ee7 100644 --- a/presets/testing/test-render-bash.yaml +++ b/presets/testing/test-render-bash.yaml @@ -2,11 +2,46 @@ name: test-render-bash title: Test Render (Bash) description: >- - No-op render simulator: sleeps per frame and can inject failures, hangs, - progress, and placeholder output. Runs a bash script; use on Linux, macOS, - or any worker with bash. A test/QA job, not a real renderer. Keep - Sleep (seconds) under 120 (the per-task timeout guardrail); Hang Frames - sleep past it to exercise the timeout path. + Simulates a render on any worker without a renderer installed, sleeping per + frame and optionally failing or hanging chosen frames. +readme: | + # Test Render (bash) + + A no-op render simulator for exercising a farm end to end without installing + any DCC application. Each task sleeps for a configurable time, optionally + prints progress, optionally writes a placeholder file, and then succeeds - or + fails, or hangs, if you asked it to. + + Use it to check scheduling, retries, timeouts, cancellation, log streaming and + progress reporting on a farm that has no renderer on it yet. + + ## Parameters + + - **Frame Range** - the frames to simulate, in OpenJD range syntax. One task per + frame. + - **Sleep Seconds** - how long each frame pretends to work before succeeding. + - **Emit Progress** - whether each task prints OpenJD progress messages, so the + progress bar in the web UI has something to show. + - **Fail Frames** - a range of frames that exit non-zero instead of succeeding, + for exercising retries and failure reporting. Empty means none fail. + - **Hang Frames** - a range of frames that stop responding instead of finishing, + for exercising task timeouts and cancellation. Empty means none hang. + - **Hang Seconds** - how long a hung frame stays hung before giving up. + - **Output Directory** - where each frame writes its placeholder file. Left + empty, nothing is written. + + ## How work is divided + + One task per frame, with no chunking. The template declares `SQI_PATH_TRANSLATION`, so the output directory is + translated per worker platform - which makes this preset useful for checking + path mapping on a mixed farm. + + ## Requirements + + No capability tag and no renderer: that is the point. These presets exist to + exercise the farm itself, so they run on any worker that can execute their + script. This is the bash variant, for Linux and macOS + workers; use the PowerShell variant on Windows. category: Testing version: 1.0.0 template: diff --git a/presets/testing/test-render-powershell.yaml b/presets/testing/test-render-powershell.yaml index 4f59ea1d..505f5187 100644 --- a/presets/testing/test-render-powershell.yaml +++ b/presets/testing/test-render-powershell.yaml @@ -2,11 +2,46 @@ name: test-render-powershell title: Test Render (PowerShell) description: >- - No-op render simulator: sleeps per frame and can inject failures, hangs, - progress, and placeholder output. Runs a PowerShell script; for Windows - workers. A test/QA job, not a real renderer. Keep Sleep (seconds) under 120 - (the per-task timeout guardrail); Hang Frames sleep past it to exercise the - timeout path. + Simulates a render on any Windows worker without a renderer installed, + sleeping per frame and optionally failing or hanging chosen frames. +readme: | + # Test Render (PowerShell) + + A no-op render simulator for exercising a farm end to end without installing + any DCC application. Each task sleeps for a configurable time, optionally + prints progress, optionally writes a placeholder file, and then succeeds - or + fails, or hangs, if you asked it to. + + Use it to check scheduling, retries, timeouts, cancellation, log streaming and + progress reporting on a farm that has no renderer on it yet. + + ## Parameters + + - **Frame Range** - the frames to simulate, in OpenJD range syntax. One task per + frame. + - **Sleep Seconds** - how long each frame pretends to work before succeeding. + - **Emit Progress** - whether each task prints OpenJD progress messages, so the + progress bar in the web UI has something to show. + - **Fail Frames** - a range of frames that exit non-zero instead of succeeding, + for exercising retries and failure reporting. Empty means none fail. + - **Hang Frames** - a range of frames that stop responding instead of finishing, + for exercising task timeouts and cancellation. Empty means none hang. + - **Hang Seconds** - how long a hung frame stays hung before giving up. + - **Output Directory** - where each frame writes its placeholder file. Left + empty, nothing is written. + + ## How work is divided + + One task per frame, with no chunking. The template declares `SQI_PATH_TRANSLATION`, so the output directory is + translated per worker platform - which makes this preset useful for checking + path mapping on a mixed farm. + + ## Requirements + + No capability tag and no renderer: that is the point. These presets exist to + exercise the farm itself, so they run on any worker that can execute their + script. This is the PowerShell variant and gates on + Windows workers; use the bash variant on Linux and macOS. category: Testing version: 1.0.0 template: diff --git a/presets/testing/test-steps-bash.yaml b/presets/testing/test-steps-bash.yaml index b8609f0a..b643df3e 100644 --- a/presets/testing/test-steps-bash.yaml +++ b/presets/testing/test-steps-bash.yaml @@ -2,9 +2,47 @@ name: test-steps-bash title: Test Steps (Bash) description: >- - Multi-step no-op job (render then publish then notify) that exercises step - dependency gating: publish waits for all render tasks; notify waits for both - render and publish. Runs bash scripts. A test/QA job, not a real renderer. + Simulates a three-step pipeline - render, publish, notify - so step + dependencies and ordering can be exercised without a renderer. +readme: | + # Test Steps (bash) + + A no-op multi-step job for exercising step dependencies. It defines three + steps - render, publish and notify - that run in order, each waiting for the + one before it to finish completely. + + Use it to check that dependent steps stay blocked until their upstream step + completes, that a failure partway through stops the steps below it, and that + the job graph renders correctly in the web UI. + + ## Parameters + + - **Frame Range** - the frames to simulate, in OpenJD range syntax. One task per + frame. + - **Sleep Seconds** - how long each frame pretends to work before succeeding. + - **Emit Progress** - whether each task prints OpenJD progress messages, so the + progress bar in the web UI has something to show. + - **Fail Frames** - a range of frames that exit non-zero instead of succeeding, + for exercising retries and failure reporting. Empty means none fail. + - **Hang Frames** - a range of frames that stop responding instead of finishing, + for exercising task timeouts and cancellation. Empty means none hang. + - **Hang Seconds** - how long a hung frame stays hung before giving up. + - **Output Directory** - where each frame writes its placeholder file. Left + empty, nothing is written. + + ## How work is divided + + The render step runs one task per frame. The publish and notify steps each run + a single task, gated on the step before. The template declares `SQI_PATH_TRANSLATION`, so the output directory is + translated per worker platform - which makes this preset useful for checking + path mapping on a mixed farm. + + ## Requirements + + No capability tag and no renderer: that is the point. These presets exist to + exercise the farm itself, so they run on any worker that can execute their + script. This is the bash variant, for Linux and macOS + workers; use the PowerShell variant on Windows. category: Testing version: 1.0.0 template: diff --git a/presets/testing/test-steps-powershell.yaml b/presets/testing/test-steps-powershell.yaml index 306cc1cc..0d9ea781 100644 --- a/presets/testing/test-steps-powershell.yaml +++ b/presets/testing/test-steps-powershell.yaml @@ -2,10 +2,47 @@ name: test-steps-powershell title: Test Steps (PowerShell) description: >- - Multi-step no-op job (render then publish then notify) that exercises step - dependency gating: publish waits for all render tasks; notify waits for both - render and publish. Runs PowerShell scripts; for Windows workers. A test/QA - job, not a real renderer. + Simulates a three-step pipeline - render, publish, notify - on Windows + workers, so step dependencies can be exercised without a renderer. +readme: | + # Test Steps (PowerShell) + + A no-op multi-step job for exercising step dependencies. It defines three + steps - render, publish and notify - that run in order, each waiting for the + one before it to finish completely. + + Use it to check that dependent steps stay blocked until their upstream step + completes, that a failure partway through stops the steps below it, and that + the job graph renders correctly in the web UI. + + ## Parameters + + - **Frame Range** - the frames to simulate, in OpenJD range syntax. One task per + frame. + - **Sleep Seconds** - how long each frame pretends to work before succeeding. + - **Emit Progress** - whether each task prints OpenJD progress messages, so the + progress bar in the web UI has something to show. + - **Fail Frames** - a range of frames that exit non-zero instead of succeeding, + for exercising retries and failure reporting. Empty means none fail. + - **Hang Frames** - a range of frames that stop responding instead of finishing, + for exercising task timeouts and cancellation. Empty means none hang. + - **Hang Seconds** - how long a hung frame stays hung before giving up. + - **Output Directory** - where each frame writes its placeholder file. Left + empty, nothing is written. + + ## How work is divided + + The render step runs one task per frame. The publish and notify steps each run + a single task, gated on the step before. The template declares `SQI_PATH_TRANSLATION`, so the output directory is + translated per worker platform - which makes this preset useful for checking + path mapping on a mixed farm. + + ## Requirements + + No capability tag and no renderer: that is the point. These presets exist to + exercise the farm itself, so they run on any worker that can execute their + script. This is the PowerShell variant and gates on + Windows workers; use the bash variant on Linux and macOS. category: Testing version: 1.0.0 template: From 59846ab36ea2e3ebe9674255e0a6511533726559 Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Tue, 18 Aug 2026 22:01:21 -0700 Subject: [PATCH 08/10] docs(presets): correct the blender output path claim for an empty value --- presets/sqi/blender-batch-render.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/presets/sqi/blender-batch-render.yaml b/presets/sqi/blender-batch-render.yaml index d7ff4093..2d433cf6 100644 --- a/presets/sqi/blender-batch-render.yaml +++ b/presets/sqi/blender-batch-render.yaml @@ -18,8 +18,9 @@ readme: | - **Frames** - the frame range, in OpenJD range syntax, e.g. `1-100` or `1-100:2` for every second frame. - **Output Path** - the output path passed to Blender as `-o`. Blender treats - it as a prefix and appends the frame number and extension. Left empty, the - path saved in the blend file is used. + it as a prefix and appends the frame number and extension. The flag is + always passed, so an empty value reaches Blender as an empty path rather + than being omitted; set it explicitly. ## How work is divided From c505a5b2157ac94b9e010446e4f22966a2e37fbb Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Tue, 18 Aug 2026 22:27:34 -0700 Subject: [PATCH 09/10] fix(python-sdk): stop the reconnect-throttle test hijacking every sleep in the process --- clients/python/tests/test_events.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/clients/python/tests/test_events.py b/clients/python/tests/test_events.py index b5633a08..cad52b57 100644 --- a/clients/python/tests/test_events.py +++ b/clients/python/tests/test_events.py @@ -14,6 +14,7 @@ import json import sys import threading +import time from collections.abc import Callable, Iterator from typing import Any @@ -271,7 +272,22 @@ def handler(ws: ServerConnection) -> None: class _Stop(Exception): pass + real_sleep = time.sleep + def fake_sleep(seconds: float) -> None: + # setattr on "sqi_client.events.time.sleep" patches the stdlib time + # module object itself, so EVERY time.sleep in the process lands here -- + # including websockets' own server.shutdown(), which sleeps + # SHUTDOWN_POLLING_INTERVAL while waiting for the serving thread to + # stop. Raising unconditionally therefore threw _Stop out of the + # run_ws_server teardown, outside the pytest.raises block, in roughly + # 8% of runs (5/60 locally) -- and appended a stray 0.1 to sleeps. + # + # Hijack only the first call, which is the throttle this test is about, + # and let every later sleep behave normally. + if sleeps: + real_sleep(seconds) + return sleeps.append(seconds) raise _Stop # break the otherwise-endless flap loop after one throttle From 186022da22343dd8f4ce85778777e2d5787ecf6d Mon Sep 17 00:00:00 2001 From: Robin Scher Date: Tue, 18 Aug 2026 22:52:09 -0700 Subject: [PATCH 10/10] fix(presets): restore the bash segment-transcode join invocation for git-bash --- .../sqi/ffmpeg-segment-transcode-bash.yaml | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/presets/sqi/ffmpeg-segment-transcode-bash.yaml b/presets/sqi/ffmpeg-segment-transcode-bash.yaml index c8aa847e..afbe0023 100644 --- a/presets/sqi/ffmpeg-segment-transcode-bash.yaml +++ b/presets/sqi/ffmpeg-segment-transcode-bash.yaml @@ -141,11 +141,27 @@ template: - name: join type: TEXT runnable: true + # LF is explicit, not incidental. This script may run under git-bash + # on Windows, where a CRLF body fails at the first line with + # "$'\r': command not found" rather than anything diagnosable. + endOfLine: LF data: | #!/usr/bin/env bash set -euo pipefail out="$1" + + # Windows (git-bash): a drive-letter or UNC path arrives with + # backslashes, which are not separators to dirname/basename or to + # the glob below, so every path operation here would silently + # produce garbage. Fold them to forward slashes, which bash, the + # glob, and ffmpeg.exe all accept on Windows. Guarded on the + # Windows path prefixes so a POSIX path that legitimately contains + # a backslash is left untouched. + case "$out" in + [A-Za-z]:[\\/]* | \\\\*) out="${out//\\//}" ;; + esac + dir="$(dirname "$out")" base="$(basename "$out")" stem="${base%.*}" @@ -160,8 +176,16 @@ template: # Zero-padded slice numbers make lexical order numeric order, so # the glob above is already in the right sequence. - list="$(mktemp)" + # + # The list is written beside the output rather than in $TMPDIR + # because ffmpeg is a native binary: under git-bash an mktemp path + # like /tmp/tmp.XXXX reaches ffmpeg.exe through MSYS argument + # conversion, and the name below is already host-native. It cannot + # collide with the glob above, which requires "_seg_" and the + # output's own extension. + list="${dir}/${stem}_concat.txt" trap 'rm -f "$list"' EXIT + : > "$list" for s in "${segs[@]}"; do printf "file '%s'\n" "$s" >> "$list" done @@ -170,5 +194,10 @@ template: rm -f "${segs[@]}" actions: onRun: - command: "{{Task.File.join}}" - args: ["{{Param.OutputFile}}"] + # bash is invoked explicitly rather than relying on the shebang: + # executing a shebang script directly is a POSIX kernel feature + # (binfmt_script) with no Windows equivalent, so "command: bash" + # is what lets this variant run under git-bash on Windows as well + # as on Linux and macOS. + command: bash + args: ["{{Task.File.join}}", "{{Param.OutputFile}}"]