feat!: opt-in broker authentication and per-worker transport identity

.github/workflows/ci.yml

@@ -149,12 +149,13 @@ jobs:
       # branch that had touched none of the reported code. Bump this
       # deliberately, in its own commit, with the fallout fixed alongside it.
       #
-      # 2.13.1 is also a FLOOR, not just a pin: .golangci.yml excludes
+      # There is also a FLOOR below this pin: .golangci.yml excludes
       # errors.AsType from errcheck by function name, and errcheck before
-      # 2.13.1 cannot resolve a generic function's name, so on an older
+      # 2.13.0 cannot resolve a generic function's name, so on an older
       # golangci-lint that exclusion silently fails to match and every
-      # errors.AsType call site is reported. Keep docs/development.md's
-      # required-version note in step with this value.
+      # errors.AsType call site is reported. The floor is 2.13.0 (what Homebrew
+      # ships, verified clean on this repo); this pin is one patch above it.
+      # Keep docs/development.md's required-version note in step with both.
       #
       # Explicit path patterns mirror the Makefile LINT_PKGS variable: they
       # exclude web/node_modules/ (third-party JS packages that happen to
@@ -959,6 +960,105 @@ jobs:
           fi
           exit $missing
 
+  # ── Integration suite (whole package) ────────────────────────────────────────
+  # Runs every file under test/integration/ via `make test-integration`, rather
+  # than naming individual tests in a -run regex the way the LDAP, OIDC, and
+  # isolation jobs below do. Those three name their tests because each needs
+  # its own runner matrix or container prerequisites; everything else in the
+  # package — broker auth, EXPR parameter binding and end-to-end execution,
+  # failure-reason propagation (both in-process and against a real worker
+  # binary), retry/auto-park behavior, and product retry-override submission —
+  # shares no such prerequisite, so a single whole-package run covers a new
+  # test the moment it is added instead of only if someone also remembers to
+  # list it in a job. `go build`s a real sqi-worker binary along the way (see
+  # worker_binary_test.go); no other tooling is required beyond Go itself.
+  #
+  # LDAP, OIDC, and isolation tests live in this same package and this job
+  # exercises them too, but ubuntu-latest ships Docker so they are expected to
+  # run rather than skip; the LDAP and Keycloak images are pulled explicitly
+  # first so a registry hiccup fails here with a clear message instead of
+  # inside the test as a cold-pull timeout that would read as a skip. The
+  # isolation tests, in contrast, require running as root (see
+  # isolation-integration below) and are expected to skip on this ordinary
+  # runner — that is fine, and this job asserts nothing about them by name.
+  #
+  # IMPORTANT: a green `go test` here proves nothing on its own if the whole
+  # suite silently collected zero tests (a build-tag regression, a stray
+  # -run filter). This job does not trust the exit code alone: it requires a
+  # test named from every file in the package that has no dedicated CI job of
+  # its own — one exception is authfixture_test.go, which holds only helpers
+  # shared by ldap_test.go and oidc_test.go and declares no test function of
+  # its own to name. A confirmed-skip line for LDAP/OIDC/isolation does not
+  # fail the job; only a missing PASS among the required names, or a nonzero
+  # exit, does.
+  integration-tests:
+    name: Integration suite
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+
+      # Pulled explicitly for the same reason ldap-integration and
+      # oidc-integration below pull them: a registry problem fails here with a
+      # clear message rather than surfacing inside the test as a cold-pull
+      # timeout, which the test's own Docker-unavailable handling would read
+      # as an ordinary skip.
+      - name: Pull OpenLDAP image
+        run: docker pull osixia/openldap:1.5.0
+
+      - name: Pull Keycloak image
+        run: docker pull quay.io/keycloak/keycloak:26.0.7
+
+      - name: Run the integration suite
+        run: |
+          set +e
+          # go test's own default (10m per package) leaves little headroom
+          # once the LDAP and Keycloak containers are in the run: up to ~90s
+          # for LDAP readiness plus up to ~180s for Keycloak, on top of the
+          # suite's own runtime, which is dominated by that container
+          # readiness rather than by the tests themselves. 15m is set
+          # explicitly here, through INTEGRATION_TEST_FLAGS, so a slow runner
+          # gets a clear failure instead of a default timeout landing close
+          # enough to the estimate to be a coin flip; local runs are
+          # unaffected since the Makefile leaves this empty by default.
+          make test-integration INTEGRATION_TEST_FLAGS="-timeout 15m" 2>&1 | tee integration-output.log
+          status=${PIPESTATUS[0]}
+          set -e
+
+          # One name from each file in test/integration/ that has no dedicated
+          # CI job of its own, so a build-tag or filter regression that let the
+          # suite silently collect zero tests cannot pass this job.
+          expected=(
+            TestDefaultConfig_NoBrokerAuth
+            TestRevocation_DisconnectsAndReclaims
+            TestEnrollment_ConnectsToRunningBrokerWithoutRestart
+            TestEXPRParamTypes_BindAndCarry
+            TestEXPRJobEndToEnd
+            TestTaskFailureReason_VisibleEndToEnd
+            TestWorkerBinaryStagingFailureReason
+            TestProductSubmit_RetryOverrides
+            TestAutoRetry_RetryThenSucceed
+          )
+          missing=0
+          for name in "${expected[@]}"; do
+            if ! grep -q -- "--- PASS: $name" integration-output.log; then
+              echo "::error::$name did not pass"
+              missing=1
+            fi
+          done
+          if [ "$status" -ne 0 ] || [ "$missing" -ne 0 ]; then
+            echo "::error::the integration suite did not all pass (exit=$status)."
+            exit 1
+          fi
+          echo "Confirmed all ${#expected[@]} named integration tests passed, and the whole suite exited 0."
+
   # Runs the official OpenJD conformance suite (a pinned submodule under
   # third_party/) against internal/openjd. Tagged `conformance`, so it does not
   # run in the default `make test`.

.golangci.yml

@@ -121,7 +121,7 @@ linters:
         # and reports it. Exclude the function rather than turning check-blank
         # off, so a genuine `_ = f()` is still caught.
         #
-        # Requires golangci-lint >= 2.13.1: earlier errcheck cannot resolve a
+        # Requires golangci-lint >= 2.13.0: earlier errcheck cannot resolve a
         # generic function's name (it reports "Error return value is not
         # checked" with no name), so this entry never matches and every
         # errors.AsType call site is flagged.

Makefile

@@ -200,8 +200,12 @@ test-cover-html: test-cover ## Open HTML coverage report in the browser
 	go tool cover -html=$(COVERAGE_OUT)
 
 .PHONY: test-integration
+# INTEGRATION_TEST_FLAGS is empty by default so a local run gets go test's
+# ordinary defaults; CI sets it to add a timeout sized for the suite's own
+# LDAP/Keycloak containers without changing what a local run does.
+INTEGRATION_TEST_FLAGS ?=
 test-integration: ## Run integration tests (tagged 'integration')
-	go test $(TEST_FLAGS) -tags integration ./test/...
+	go test $(TEST_FLAGS) -tags integration -v $(INTEGRATION_TEST_FLAGS) ./test/...
 
 .PHONY: test-conformance
 test-conformance: ## Run the official OpenJD conformance suite (needs the pinned submodule)

ROADMAP.md

@@ -79,7 +79,7 @@ Configuration cascades: farm defaults → queue overrides, with retry policy (ma
 
 **Scheduling considers:** job priority, task dependencies, queue and farm policy (concurrency limits, scheduling mode), compute location affinity, worker capability tags (OS, GPU, installed software), and usage pool availability.
  - *Design:* ready tasks remain `ready` until a worker sends a core-NATS
-    request to `work.lease.<queue>`. The server computes free cores
+    request to `work.lease.<worker>.<queue>`. The server computes free cores
     (`CPUCount − Σ committed`), selects a priority-ordered batch that fits,
     atomically transitions the batch `ready → assigned` (stamping `assigned_at`
     only now), and replies. The `SQI_WORK` JetStream stream, `work.assign.<queue>`
@@ -212,7 +212,7 @@ NATS JetStream handles:
 - Heartbeats and worker registration
 
 Work leases use **core NATS** request/reply (not JetStream): the worker requests
-work on `work.lease.<queue>` and the server replies with a batch it is
+work on `work.lease.<worker>.<queue>` and the server replies with a batch it is
 authorized to run (pull-based). Real-time UI updates reach web clients over
 WebSocket, fanned out by the server after it ingests the JetStream messages.
 

cliff.toml

@@ -38,7 +38,9 @@ body = """
 {% for commit in commits %}
 - {% if commit.scope %}**{{ commit.scope }}:** {% endif %}\
 {{ commit.message | upper_first }}\
-{% if commit.breaking %} ⚠️ BREAKING{% endif %} \
+{% if commit.breaking %} ⚠️ BREAKING\
+{% if commit.breaking_description and commit.breaking_description != commit.message %} — {{ commit.breaking_description }}{% endif %}\
+{% endif %} \
 ([{{ commit.id | truncate(length=7, end="") }}](https://github.com/uberware/sqi/commit/{{ commit.id }}))
 {%- endfor %}
 {% endfor %}\n

cmd/sqi-server/backup.go

@@ -31,6 +31,12 @@ The backup is produced using SQLite's VACUUM INTO statement, which snapshots
 the live database without taking an exclusive lock. The server may be running
 or stopped — either works. The destination file must not already exist.
 
+The source database path defaults to store.sqlite_path from the resolved
+configuration (the root -c/--config file and SQI_STORE_SQLITE_PATH), falling
+back to the legacy SQI_SQLITE_PATH environment variable and then to "sqi.db".
+Pass --db to override it explicitly. The source database must already exist —
+this command never creates one.
+
 Example:
   sqi-server backup --db sqi.db --out sqi-backup-$(date +%Y%m%d).db`,
 	RunE: runBackup,
@@ -39,8 +45,8 @@ Example:
 func init() {
 	backupCmd.Flags().StringVar(
 		&backupFlags.DBPath,
-		"db", envOr("SQI_SQLITE_PATH", "sqi.db"),
-		"path to source SQLite database file",
+		"db", "sqi.db",
+		"path to source SQLite database file (defaults to store.sqlite_path from config, then the deprecated SQI_SQLITE_PATH, then \"sqi.db\")",
 	)
 	backupCmd.Flags().StringVar(
 		&backupFlags.OutPath,
@@ -52,20 +58,31 @@ func init() {
 	}
 }
 
-func runBackup(_ *cobra.Command, _ []string) error {
-	if backupFlags.DBPath == "" {
-		return errors.New("source database path is empty; use --db or set SQI_SQLITE_PATH")
-	}
+func runBackup(cmd *cobra.Command, _ []string) error {
 	if backupFlags.OutPath == "" {
 		return errors.New("destination path is empty; use --out")
 	}
 
+	dbPath, err := resolveDBPath(backupFlags.DBPath, cmd != nil && cmd.Flags().Changed("db"))
+	if err != nil {
+		return err
+	}
+	if dbPath == "" {
+		return errors.New("source database path is empty; use --db, set store.sqlite_path, or set SQI_STORE_SQLITE_PATH")
+	}
+	if err := requireExistingDB(dbPath); err != nil {
+		return err
+	}
+	if err := requireMigratedDB(dbPath); err != nil {
+		return err
+	}
+
 	logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
 
 	ctx := context.Background()
 
-	logger.InfoContext(ctx, "backup: opening source database", slog.String("path", backupFlags.DBPath))
-	st, err := sqlite.Open(ctx, backupFlags.DBPath, sqlite.Options{AutoMigrate: false})
+	logger.InfoContext(ctx, "backup: opening source database", slog.String("path", dbPath))
+	st, err := sqlite.Open(ctx, dbPath, sqlite.Options{AutoMigrate: false})
 	if err != nil {
 		return fmt.Errorf("open source database: %w", err)
 	}
@@ -74,7 +91,7 @@ func runBackup(_ *cobra.Command, _ []string) error {
 	start := time.Now()
 	logger.InfoContext(
 		ctx, "backup: starting",
-		slog.String("src", backupFlags.DBPath),
+		slog.String("src", dbPath),
 		slog.String("dst", backupFlags.OutPath),
 	)
 

cmd/sqi-server/dbpath.go

@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"os"
+
+	_ "modernc.org/sqlite" // register "sqlite" driver
+
+	"github.com/uberware/sqi/internal/config"
+)
+
+// resolveDBPath applies the database-path precedence shared by backup,
+// migrate, and worker, highest priority first:
+//
+//  1. explicit is used as-is when explicitChanged is true (the operator
+//     passed --db on the command line).
+//  2. The config layer: the root -c/--config file and SQI_STORE_SQLITE_PATH,
+//     i.e. whatever [config.LoadWithSources] resolves store.sqlite_path to,
+//     when [config.Sources.StoreSQLitePath] reports the file or env var
+//     actually set it — NOT when the resolved value merely differs from the
+//     built-in default, which a config file that restates the default
+//     value (as config/sqi-server.example.yaml does) would defeat.
+//  3. The legacy SQI_SQLITE_PATH environment variable, kept working as an
+//     alias. A deprecation notice naming SQI_STORE_SQLITE_PATH — the
+//     variable sqi-server itself reads — is printed to stderr when this is
+//     the layer that decided the path.
+//  4. The built-in default ("sqi.db").
+//
+// The config layer is always loaded, even when an explicit --db makes its
+// result moot, so a malformed --config file is reported as an error rather
+// than silently ignored when the operator asked for a specific one (root
+// -c/--config was passed explicitly). Without an explicit -c, a config-load
+// failure — an auto-discovered but broken /etc/sqi/sqi-server.yaml, or an
+// unrelated malformed SQI_* env var with nothing to do with the database
+// path — is NOT a hard failure: backup, migrate, and worker are the tools
+// reached for when something is already broken, so a warning goes to
+// stderr and resolution falls through to the legacy env var and default as
+// if the config layer had decided nothing.
+func resolveDBPath(explicit string, explicitChanged bool) (string, error) {
+	cfg, src, err := config.LoadWithSources(persistentFlags.ConfigFile, config.FlagOverrides{})
+	if err != nil {
+		if persistentFlags.ConfigFile != "" {
+			return "", fmt.Errorf("load config: %w", err)
+		}
+		fmt.Fprintf(
+			os.Stderr,
+			"warning: could not load configuration (%v); falling back to SQI_SQLITE_PATH or the built-in default\n",
+			err,
+		)
+		cfg = config.DefaultConfig()
+		src = config.Sources{}
+	}
+
+	if explicitChanged {
+		return explicit, nil
+	}
+
+	if src.StoreSQLitePath {
+		return cfg.Store.SQLitePath, nil
+	}
+
+	if legacy := envOr("SQI_SQLITE_PATH", ""); legacy != "" {
+		fmt.Fprintln(os.Stderr,
+			"warning: SQI_SQLITE_PATH is deprecated; set SQI_STORE_SQLITE_PATH instead, "+
+				"which is the variable sqi-server itself reads")
+		return legacy, nil
+	}
+
+	return cfg.Store.SQLitePath, nil
+}
+
+// requireExistingDB stats path and returns an actionable error naming it and
+// how to point elsewhere when no file exists there. Used by backup and
+// worker, which must never create a database — unlike migrate, whose job is
+// to create one.
+func requireExistingDB(path string) error {
+	if _, err := os.Stat(path); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf(
+				"no database at %s; point --db, store.sqlite_path, or SQI_STORE_SQLITE_PATH at the right file, "+
+					"or run \"sqi-server migrate up\" to create one there",
+				path,
+			)
+		}
+		return fmt.Errorf("stat %s: %w", path, err)
+	}
+	return nil
+}
+
+// requireMigratedDB catches the case requireExistingDB's stat check cannot
+// see: a file that exists at path but was never migrated (empty, or created
+// by something other than "migrate up" or the server's own AutoMigrate).
+// sqlite.Open with AutoMigrate: false succeeds against such a file — SQLite
+// opens an empty database happily — so the first real query would otherwise
+// fail with a raw driver error ("no such table: ..."). Checked directly
+// against sqlite_master rather than through the store, so it runs before
+// any store method that would surface that error unremediated.
+func requireMigratedDB(path string) error {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return fmt.Errorf("open %s: %w", path, err)
+	}
+	defer db.Close()
+
+	var count int
+	err = db.QueryRowContext(
+		context.Background(),
+		`SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`,
+	).Scan(&count)
+	if err != nil {
+		return fmt.Errorf("check schema at %s: %w", path, err)
+	}
+	if count == 0 {
+		return fmt.Errorf("the database at %s has no schema; run \"sqi-server migrate up\" first", path)
+	}
+	return nil
+}