diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d252240a..b7faeb7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -510,7 +510,7 @@ jobs: # Reuses scripts/smoke.sh (the same script developers run via `make smoke`). # Passing the prebuilt binaries skips its internal `make build`, so no GNU # make is required on the Windows runner. - - name: End-to-end smoke (REST + WebSocket) + - name: End-to-end smoke (REST + WebSocket, plain / broker-auth / TLS) run: | ext=""; [ "$RUNNER_OS" = "Windows" ] && ext=".exe" export SQI_SERVER_BIN="bin/sqi-server${ext}" @@ -519,7 +519,24 @@ jobs: # status is not masked by `export` (shellcheck SC2155). SQI_SMOKE_PYTHON="$(command -v python)" export SQI_SMOKE_PYTHON - bash scripts/smoke.sh + bash scripts/smoke.sh 2>&1 | tee smoke-output.log + status=${PIPESTATUS[0]} + + # The script runs three modes. Assert each one actually reported a + # pass by name: a filter or branch regression that silently skipped a + # mode would otherwise leave this job green while covering less. + missing=0 + for mode in noauth brokerauth tls; do + if ! grep -q "SMOKE TEST PASSED (mode=$mode)" smoke-output.log; then + echo "::error::smoke mode $mode did not report a pass" + missing=1 + fi + done + if [ "$status" -ne 0 ] || [ "$missing" -ne 0 ]; then + echo "::error::the smoke test did not pass in all three modes (exit=$status)." + exit 1 + fi + echo "Confirmed all three smoke modes passed." # ── Docker smoke test ──────────────────────────────────────────────────────── # Builds both Docker images from source (no registry push) and runs them @@ -1045,6 +1062,8 @@ jobs: TestWorkerBinaryStagingFailureReason TestProductSubmit_RetryOverrides TestAutoRetry_RetryThenSucceed + TestTLSEndToEnd + TestTLSWorkerWithoutCAIsRefused ) missing=0 for name in "${expected[@]}"; do @@ -1059,6 +1078,102 @@ jobs: fi echo "Confirmed all ${#expected[@]} named integration tests passed, and the whole suite exited 0." + # ── mDNS discovery over real multicast ─────────────────────────────────────── + # A SEPARATE job, not folded into integration-tests, for two reasons. Whether + # a hosted runner can do multicast on loopback is the one thing about this + # suite that could not be verified before it was written, so isolating it + # means such a runner fails this job precisely instead of taking the whole + # integration suite down with an unrelated-looking error. And the real-binary + # test binds the test broker to all interfaces, which should happen only when + # discovery testing was asked for — `make test-integration` skips it. + # + # SQI_TEST_REQUIRE_MULTICAST=1 (set by `make test-discovery`) turns the + # capability skip into a failure, so this job cannot go green while covering + # nothing — the same guard ldap-integration and isolation-integration use. + discovery-integration: + name: mDNS discovery (real multicast) + runs-on: ubuntu-latest + needs: test + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # The advertisement carries ".local", which the worker dials. + # macOS answers that through mDNSResponder; a stock ubuntu runner has no + # mDNS NSS module, so the worker would discover the server and then fail + # to dial it. + # + # A hosts entry rather than libnss-mdns + avahi: avahi-daemon would want + # port 5353, which is the port the test's own responder binds, so + # installing it to fix resolution risks breaking advertisement. Mapping + # the name to loopback needs no daemon and cannot conflict. + - name: Resolve this host's mDNS name locally + run: | + echo "127.0.0.1 $(hostname).local" | sudo tee -a /etc/hosts + getent hosts "$(hostname).local" || { + echo "::error::$(hostname).local still does not resolve"; exit 1; } + + # Two things a Linux loopback lacks that macOS lo0 has by default, and the + # loopback-only advertisement needs both. The tests advertise on loopback + # only — deliberately, so a run never announces a service on the network it + # happens to be on — and refuse to fall back to a real interface, so + # neither of these can be skipped: without them the suite cannot run at all. + # + # 1. The MULTICAST flag. Linux `lo` ships without it, and the tests select + # their interfaces by that flag, so an untouched `lo` looks to them like + # a host that cannot do multicast. + # 2. A non-loopback address. zeroconf fills the advertisement's address + # records from the interface's own addresses and discards loopback ones + # (addrsForInterface: `!ipnet.IP.IsLoopback()`); a Linux `lo` carries + # only 127.0.0.1 and ::1, so registering on loopback alone fails + # outright with "Could not determine host IP addresses". macOS lo0 also + # carries fe80::1, which is the whole reason this never shows up there. + # Adding that same link-local address makes the two behave alike — it + # is scoped to the link and is not routed anywhere. + - name: Prepare loopback for mDNS + run: | + ip -o addr show lo + sudo ip link set lo multicast on + ip link show lo | grep -q MULTICAST || { + echo "::error::could not enable MULTICAST on lo"; exit 1; } + sudo ip -6 addr add fe80::1/64 dev lo || true + ip -o -6 addr show lo scope link | grep -q fe80 || { + echo "::error::lo has no link-local address, so a loopback-only mDNS" + echo "::error::advertisement cannot register — see the comment above" + exit 1; } + + - name: Run the discovery suite over real multicast + run: | + set +e + make test-discovery 2>&1 | tee discovery-output.log + status=${PIPESTATUS[0]} + set -e + + expected=( + TestDiscovery_TLSRecordsCrossTheWire + TestDiscovery_AdvertisedBrokerTLSReachesWorkerConfig + TestDiscovery_RealBinaryFindsItsServerOverMDNS + ) + missing=0 + for name in "${expected[@]}"; do + if ! grep -q -- "--- PASS: $name" discovery-output.log; then + echo "::error::$name did not pass" + missing=1 + fi + done + if [ "$status" -ne 0 ] || [ "$missing" -ne 0 ]; then + echo "::error::the mDNS discovery suite did not pass (exit=$status)." + echo "::error::if the runner cannot do multicast, the suite says so explicitly." + exit 1 + fi + echo "Confirmed all ${#expected[@]} discovery tests passed over real multicast." + # Runs the official OpenJD conformance suite (a pinned submodule under # third_party/) against internal/openjd. Tagged `conformance`, so it does not # run in the default `make test`. diff --git a/Makefile b/Makefile index 54fa0ba4..63e295a0 100644 --- a/Makefile +++ b/Makefile @@ -335,6 +335,17 @@ test-isolation: ## Run run-as-user isolation tests as root against real OS accou docker run --rm --init sqi-isolation-test \ go test $(TEST_FLAGS) -tags integration -run 'TestIsolation_' -v -timeout 15m ./test/integration/ +.PHONY: test-discovery +test-discovery: ## Run the mDNS discovery tests over REAL multicast (fails rather than skips if multicast is unavailable) + @echo "note: advertisements are restricted to loopback, so nothing is announced" + @echo " on your network. One test (RealBinary...) binds the test broker to" + @echo " all interfaces for ~10s; make test-integration skips that one." + @echo " On Linux, loopback needs both: sudo ip link set lo multicast on" + @echo " and: sudo ip -6 addr add fe80::1/64 dev lo (zeroconf discards" + @echo " loopback addresses, so lo has nothing to advertise without it)" + SQI_TEST_REQUIRE_MULTICAST=1 go test $(TEST_FLAGS) -tags integration \ + -run 'TestDiscovery_' -v -timeout 10m ./test/integration/ + .PHONY: test-isolation-windows test-isolation-windows: ## Run windows run-as-user isolation tests as SYSTEM against real local accounts (needs an elevated shell) @powershell -NoProfile -ExecutionPolicy Bypass -File scripts/test-isolation-windows.ps1 diff --git a/README.md b/README.md index bb3265e9..e4287aa7 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ sqi-worker # run on each render node — finds the server automatically On a local network, workers discover the server via mDNS and connect without any configuration. Open a browser, start submitting. That's it. -There is no authentication in this mode — the server trusts the local network, by design. To require logins, set `auth.enabled: true` and see [docs/auth.md](docs/auth.md), which covers the first-admin bootstrap, roles, API keys, and connecting a directory or SSO provider. +There is no authentication in this mode — the server trusts the local network, by design. To require logins, set `auth.enabled: true` and see [docs/auth.md](docs/auth.md), which covers the first-admin bootstrap, roles, API keys, and connecting a directory or SSO provider. Traffic is unencrypted in this mode too; [docs/tls.md](docs/tls.md) covers turning on TLS for the API and the worker transport, which is a separate opt-in setting. See the [Quickstart](docs/quickstart.md) for a full walkthrough (binary or Docker Compose), including creating a farm and queue and submitting your first job. diff --git a/cmd/sqi-server/README.md b/cmd/sqi-server/README.md index 6a2b7d78..b30663cf 100644 --- a/cmd/sqi-server/README.md +++ b/cmd/sqi-server/README.md @@ -202,4 +202,5 @@ internal/ - [`docs/architecture.md`](../../docs/architecture.md) — Component layout and job lifecycle data flow. - [`docs/configuration.md`](../../docs/configuration.md) — Complete configuration reference. - [`docs/api.md`](../../docs/api.md) — REST API reference with worked examples. +- [`docs/tls.md`](../../docs/tls.md) — In-process TLS for the API and broker, `tls init` / `tls issue`, and worker mTLS. - [`docs/development.md`](../../docs/development.md) — Local setup and contribution guide. diff --git a/cmd/sqi-server/root.go b/cmd/sqi-server/root.go index ce50b338..4b1cf7e4 100644 --- a/cmd/sqi-server/root.go +++ b/cmd/sqi-server/root.go @@ -60,6 +60,7 @@ func init() { configCmd, backupCmd, workerCmd, + tlsCmd, ) } diff --git a/cmd/sqi-server/serve.go b/cmd/sqi-server/serve.go index b60fe49f..e78ea8f3 100644 --- a/cmd/sqi-server/serve.go +++ b/cmd/sqi-server/serve.go @@ -167,6 +167,8 @@ func runServe(cmd *cobra.Command, _ []string) error { func serverConfig(cfg config.Config, schedCfg scheduler.Config) server.Config { return server.Config{ HTTPAddr: cfg.HTTP.Addr, + HTTPTLS: cfg.HTTP.TLS, + NATSTLS: cfg.NATS.TLS, CORSOrigins: cfg.HTTP.CORSOrigins, NATSAddr: cfg.NATS.Addr, NATSAuthEnabled: cfg.NATS.Auth.Enabled, diff --git a/cmd/sqi-server/tls.go b/cmd/sqi-server/tls.go new file mode 100644 index 00000000..00853546 --- /dev/null +++ b/cmd/sqi-server/tls.go @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "time" + + "github.com/spf13/cobra" + + "github.com/uberware/sqi/internal/certgen" +) + +const ( + // caValidity is the farm CA's lifetime. Long, because rotating a CA means + // redistributing trust to every worker on the farm. + caValidity = 10 * 365 * 24 * time.Hour + + // leafValidity is the lifetime of server and client certificates. + leafValidity = 2 * 365 * 24 * time.Hour +) + +var tlsCmd = &cobra.Command{ + Use: "tls", + Short: "Generate and manage TLS certificate material", +} + +var tlsInitCmd = &cobra.Command{ + Use: "init", + Short: "Generate a farm CA and a server certificate", + Long: `Generate the certificate material a farm needs to run sqi over TLS. + +Writes a farm certificate authority (ca.crt, ca.key) and a server certificate +(server.crt, server.key) into the output directory. Copy ca.crt to every +worker; the private keys never leave the machine that generated them. + +Refuses to overwrite an existing CA: replacing one invalidates every +certificate already issued from it.`, + RunE: runTLSInit, +} + +var ( + tlsInitOut string + tlsInitHosts []string + tlsInitClients []string + + tlsIssueOut string + tlsIssueHosts []string + tlsIssueClients []string + tlsIssueForce bool +) + +var tlsIssueCmd = &cobra.Command{ + Use: "issue", + Short: "Issue certificates from an existing farm CA", + Long: `Issue additional certificates from a farm CA that already exists. + +Use this to add a worker to a farm running mutual TLS, or to rotate the server +certificate. It never touches the CA itself. + +` + "`tls init`" + ` deliberately refuses to overwrite a CA, so it cannot do either +job: a second invocation fails as a whole, taking --client with it. + +Existing files are never replaced without --force, because a client key that is +already deployed belongs to a running worker, and replacing it takes that worker +offline at its next restart with nothing to indicate why.`, + RunE: runTLSIssue, +} + +func init() { + tlsInitCmd.Flags().StringVar(&tlsInitOut, "out", "./certs", + "directory to write certificate material into") + tlsInitCmd.Flags().StringArrayVar(&tlsInitHosts, "host", nil, + "hostname or IP the server certificate must cover (repeatable; defaults to this machine's hostname plus localhost, 127.0.0.1 and ::1)") + tlsInitCmd.Flags().StringArrayVar(&tlsInitClients, "client", nil, + "worker ID to issue a client certificate for, used only with nats.tls.client_ca_file (repeatable)") + tlsCmd.AddCommand(tlsInitCmd) + + tlsIssueCmd.Flags().StringVar(&tlsIssueOut, "out", "./certs", + "directory holding ca.crt and ca.key, and where new material is written") + tlsIssueCmd.Flags().StringArrayVar(&tlsIssueClients, "client", nil, + "worker ID to issue a client certificate for (repeatable)") + tlsIssueCmd.Flags().StringArrayVar(&tlsIssueHosts, "host", nil, + "reissue the server certificate covering these hosts (repeatable; loopback is always added)") + tlsIssueCmd.Flags().BoolVar(&tlsIssueForce, "force", false, + "replace certificate files that already exist") + tlsCmd.AddCommand(tlsIssueCmd) +} + +// loadFarmCA reads the CA written by `tls init` from dir. +func loadFarmCA(dir string) (*certgen.CA, error) { + certPEM, err := os.ReadFile(filepath.Join(dir, "ca.crt")) + if err != nil { + return nil, fmt.Errorf("no farm CA in %s (%w); create one with `sqi-server tls init`", dir, err) + } + keyPEM, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + return nil, fmt.Errorf("no CA key in %s (%w); it must sit beside ca.crt — `sqi-server tls init` writes both", dir, err) + } + return certgen.LoadCA(certPEM, keyPEM) +} + +// refuseExisting reports an error when base.crt or base.key already exists and +// --force was not given. +func refuseExisting(dir, base string) error { + if tlsIssueForce { + return nil + } + for _, ext := range []string{".crt", ".key"} { + path := filepath.Join(dir, base+ext) + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("%s already exists; pass --force to replace it "+ + "(a deployed key belongs to a running worker, which stops connecting once it is replaced)", path) + } + } + return nil +} + +func runTLSIssue(cmd *cobra.Command, _ []string) error { + if len(tlsIssueClients) == 0 && len(tlsIssueHosts) == 0 { + return errors.New("nothing to issue: pass --client and/or --host ") + } + + ca, err := loadFarmCA(tlsIssueOut) + if err != nil { + return err + } + + out := cmd.OutOrStdout() + if len(tlsIssueHosts) > 0 { + hosts := resolveTLSHosts(tlsIssueHosts) + if err := refuseExisting(tlsIssueOut, "server"); err != nil { + return err + } + leaf, err := ca.NewServerCert(hosts, leafValidity) + if err != nil { + return err + } + if err := certgen.WriteLeaf(tlsIssueOut, "server", leaf); err != nil { + return err + } + fmt.Fprintf(out, "Reissued %s/server.crt covering %v\n", tlsIssueOut, hosts) + fmt.Fprintf(out, " The server reads certificates once, at startup — restart it to pick this up.\n\n") + } + + for _, id := range tlsIssueClients { + base := "client-" + id + if err := refuseExisting(tlsIssueOut, base); err != nil { + return err + } + leaf, err := ca.NewClientCert(id, leafValidity) + if err != nil { + return err + } + if err := certgen.WriteLeaf(tlsIssueOut, base, leaf); err != nil { + return err + } + fmt.Fprintf(out, "Issued %s/%s.crt for worker %q\n", tlsIssueOut, base, id) + } + + if len(tlsIssueClients) > 0 { + fmt.Fprintf(out, "\nOn each of those workers:\n") + fmt.Fprintf(out, " nats.tls_cert_file: /path/to/client-.crt\n") + fmt.Fprintf(out, " nats.tls_key_file: /path/to/client-.key\n") + fmt.Fprintf(out, " nats.tls_ca_file: /path/to/ca.crt\n") + fmt.Fprintf(out, "\nThe CA is unchanged, so existing workers keep working.\n") + } + return nil +} + +// loopbackSANs are always present on a generated server certificate. +var loopbackSANs = []string{"localhost", "127.0.0.1", "::1"} + +// resolveTLSHosts returns the SAN list for the server certificate: whatever +// --host supplied, plus this machine's hostname when nothing was supplied, +// plus loopback in every case. +// +// Loopback is appended even when --host names specific hosts. An operator who +// names only the LAN host would otherwise get a certificate that cannot be +// verified from the machine itself — which breaks a local `curl https:// +// localhost:8080/healthz`, any loopback health probe, and anything else that +// reaches the server by the name it actually runs under. Duplicates are +// dropped so an explicit --host localhost does not appear twice. +func resolveTLSHosts(flagHosts []string) []string { + hosts := slices.Clone(flagHosts) + if len(hosts) == 0 { + if h, err := os.Hostname(); err == nil && h != "" { + hosts = append(hosts, h) + } + } + for _, l := range loopbackSANs { + if !slices.Contains(hosts, l) { + hosts = append(hosts, l) + } + } + return hosts +} + +// writeTLSMaterial generates and writes the CA, the server certificate, and +// any requested client certificates. +func writeTLSMaterial(dir string, hosts, clients []string) error { + ca, err := certgen.NewCA("sqi farm CA", caValidity) + if err != nil { + return err + } + if err := certgen.WriteCA(dir, ca); err != nil { + return err + } + server, err := ca.NewServerCert(hosts, leafValidity) + if err != nil { + return err + } + if err := certgen.WriteLeaf(dir, "server", server); err != nil { + return err + } + for _, id := range clients { + leaf, err := ca.NewClientCert(id, leafValidity) + if err != nil { + return err + } + if err := certgen.WriteLeaf(dir, "client-"+id, leaf); err != nil { + return err + } + } + return nil +} + +func runTLSInit(cmd *cobra.Command, _ []string) error { + hosts := resolveTLSHosts(tlsInitHosts) + if err := writeTLSMaterial(tlsInitOut, hosts, tlsInitClients); err != nil { + return err + } + + // One template rather than a run of Fprintf calls, so the rendered layout + // is visible in the source. TestTLSInit_PrintsConfigKeys only greps for + // substrings, so a lost blank line would otherwise go unnoticed. + fmt.Fprintf(cmd.OutOrStdout(), `Wrote certificate material to %[1]s + SANs: %[2]v + +On sqi-server: + http.tls.enabled: true + http.tls.cert_file: %[1]s/server.crt + http.tls.key_file: %[1]s/server.key + nats.tls.enabled: true + nats.tls.cert_file: %[1]s/server.crt + nats.tls.key_file: %[1]s/server.key + +On each sqi-worker (copy ca.crt over first): + nats.tls_ca_file: /path/to/ca.crt + nats.server_tls_ca_file: /path/to/ca.crt +`, tlsInitOut, hosts) + + if len(tlsInitClients) > 0 { + fmt.Fprintf(cmd.OutOrStdout(), ` +For client-certificate (mTLS) workers, also set on the server: + nats.tls.client_ca_file: %s/ca.crt +and give each worker its own client-.crt / .key as + nats.tls_cert_file / nats.tls_key_file +`, tlsInitOut) + } + + fmt.Fprint(cmd.OutOrStdout(), ` +To add a worker or rotate the server certificate later, use: + sqi-server tls issue --client +This command will not run again in the same directory: it refuses to replace a +CA, because every certificate issued from the old one would stop verifying. + +Enabling TLS is a coordinated restart, not a rolling one: distribute +ca.crt and stage worker config BEFORE flipping the server. See docs/tls.md. +`) + return nil +} diff --git a/cmd/sqi-server/tls_test.go b/cmd/sqi-server/tls_test.go new file mode 100644 index 00000000..77ed3d30 --- /dev/null +++ b/cmd/sqi-server/tls_test.go @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bytes" + "crypto/x509" + "encoding/pem" + "net" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// runTLSInitCmd drives "tls init" with args and returns its stdout. +func runTLSInitCmd(t *testing.T, args ...string) (string, error) { + t.Helper() + // Reset the package-level flag targets: cobra flag vars persist between + // runs in the same process, and a stale --client would leak across tests. + tlsInitOut, tlsInitHosts, tlsInitClients = "./certs", nil, nil + + prepareRoot(append([]string{"tls", "init"}, args...)) + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + err := rootCmd.Execute() + return out.String(), err +} + +// parseCertFile reads and parses a PEM certificate from disk. +func parseCertFile(t *testing.T, path string) *x509.Certificate { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + block, _ := pem.Decode(raw) + if block == nil { + t.Fatalf("no PEM block in %s", path) + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parse %s: %v", path, err) + } + return cert +} + +func TestTLSInit_WritesExpectedFiles(t *testing.T) { + dir := filepath.Join(t.TempDir(), "certs") + if _, err := runTLSInitCmd(t, "--out", dir, "--host", "sqi.example"); err != nil { + t.Fatalf("tls init: %v", err) + } + + for _, name := range []string{"ca.crt", "ca.key", "server.crt", "server.key"} { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("missing %s: %v", name, err) + } + } + + server := parseCertFile(t, filepath.Join(dir, "server.crt")) + if !slices.Contains(server.DNSNames, "sqi.example") { + t.Errorf("DNSNames = %v, want it to contain sqi.example", server.DNSNames) + } + // Loopback is always included so the server can verify its own listener. + if !slices.Contains(server.DNSNames, "localhost") { + t.Errorf("DNSNames = %v, want it to contain localhost", server.DNSNames) + } + for _, want := range []string{"127.0.0.1", "::1"} { + if !slices.ContainsFunc(server.IPAddresses, func(ip net.IP) bool { return ip.Equal(net.ParseIP(want)) }) { + t.Errorf("IPAddresses = %v, want it to contain %s", server.IPAddresses, want) + } + } + if len(server.ExtKeyUsage) != 1 || server.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth { + t.Errorf("ExtKeyUsage = %v, want [ServerAuth]", server.ExtKeyUsage) + } + + // The leaf must verify against the CA it was issued from. + ca := parseCertFile(t, filepath.Join(dir, "ca.crt")) + if !ca.IsCA { + t.Error("ca.crt is not a CA certificate") + } + pool := x509.NewCertPool() + pool.AddCert(ca) + if _, err := server.Verify(x509.VerifyOptions{ + Roots: pool, + DNSName: "sqi.example", + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("server.crt does not verify against ca.crt: %v", err) + } + + // Private keys must not be world- or group-readable. + for _, name := range []string{"ca.key", "server.key"} { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("stat %s: %v", name, err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Errorf("%s mode = %04o, want 0600", name, got) + } + } +} + +func TestTLSInit_ClientCerts(t *testing.T) { + dir := filepath.Join(t.TempDir(), "certs") + if _, err := runTLSInitCmd(t, "--out", dir, "--client", "worker-01", "--client", "worker-02"); err != nil { + t.Fatalf("tls init: %v", err) + } + + for _, id := range []string{"worker-01", "worker-02"} { + cert := parseCertFile(t, filepath.Join(dir, "client-"+id+".crt")) + if len(cert.ExtKeyUsage) != 1 || cert.ExtKeyUsage[0] != x509.ExtKeyUsageClientAuth { + t.Errorf("%s ExtKeyUsage = %v, want [ClientAuth]", id, cert.ExtKeyUsage) + } + if cert.Subject.CommonName != id { + t.Errorf("CommonName = %q, want %q", cert.Subject.CommonName, id) + } + if _, err := os.Stat(filepath.Join(dir, "client-"+id+".key")); err != nil { + t.Errorf("missing key for %s: %v", id, err) + } + } +} + +func TestTLSInit_RefusesExistingCA(t *testing.T) { + dir := filepath.Join(t.TempDir(), "certs") + if _, err := runTLSInitCmd(t, "--out", dir); err != nil { + t.Fatalf("first tls init: %v", err) + } + original, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + t.Fatalf("read ca.key: %v", err) + } + + _, err = runTLSInitCmd(t, "--out", dir) + if err == nil { + t.Fatal("second tls init overwrote an existing CA") + } + if !strings.Contains(err.Error(), "CA already exists") { + t.Errorf("error = %v, want it to name the existing CA", err) + } + + after, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + t.Fatalf("re-read ca.key: %v", err) + } + if !bytes.Equal(after, original) { + t.Error("ca.key changed despite the refusal; every certificate issued from it would stop verifying") + } +} + +func TestTLSInit_PrintsConfigKeys(t *testing.T) { + dir := filepath.Join(t.TempDir(), "certs") + out, err := runTLSInitCmd(t, "--out", dir, "--client", "worker-01") + if err != nil { + t.Fatalf("tls init: %v", err) + } + for _, want := range []string{ + "http.tls.cert_file", + "nats.tls.cert_file", + "nats.tls_ca_file", + "nats.server_tls_ca_file", + "nats.tls.client_ca_file", + "coordinated restart", + } { + if !strings.Contains(out, want) { + t.Errorf("output does not mention %q; an operator is left guessing what to do with the files\n%s", want, out) + } + } +} + +// ── tls issue ──────────────────────────────────────────────────────────────── +// +// `tls init` deliberately refuses to overwrite a CA, which left no way to add a +// worker to an established mTLS farm or to rotate a server certificate: the +// whole command failed, taking --client with it. `tls issue` is that path. + +// runTLSIssueCmd drives "tls issue" and returns its stdout. +func runTLSIssueCmd(t *testing.T, args ...string) (string, error) { + t.Helper() + tlsIssueOut, tlsIssueClients, tlsIssueHosts, tlsIssueForce = "./certs", nil, nil, false + + prepareRoot(append([]string{"tls", "issue"}, args...)) + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + err := rootCmd.Execute() + return out.String(), err +} + +// existingFarm runs `tls init` into a fresh directory and returns it. +func existingFarm(t *testing.T) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "certs") + if _, err := runTLSInitCmd(t, "--out", dir, "--host", "sqi.example"); err != nil { + t.Fatalf("tls init: %v", err) + } + return dir +} + +func TestTLSIssue_ClientCertAgainstAnExistingCA(t *testing.T) { + dir := existingFarm(t) + caBefore, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + t.Fatalf("read ca.key: %v", err) + } + + if _, err := runTLSIssueCmd(t, "--out", dir, "--client", "render-07"); err != nil { + t.Fatalf("tls issue --client against an existing CA: %v", err) + } + + cert := parseCertFile(t, filepath.Join(dir, "client-render-07.crt")) + if len(cert.ExtKeyUsage) != 1 || cert.ExtKeyUsage[0] != x509.ExtKeyUsageClientAuth { + t.Errorf("ExtKeyUsage = %v, want [ClientAuth]", cert.ExtKeyUsage) + } + + // It must chain to the farm's EXISTING CA, or the broker rejects it. + ca := parseCertFile(t, filepath.Join(dir, "ca.crt")) + pool := x509.NewCertPool() + pool.AddCert(ca) + if _, err := cert.Verify(x509.VerifyOptions{ + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }); err != nil { + t.Errorf("issued certificate does not verify against the farm CA: %v", err) + } + + // Issuing must never disturb the CA itself. + caAfter, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + t.Fatalf("re-read ca.key: %v", err) + } + if !bytes.Equal(caBefore, caAfter) { + t.Error("ca.key changed while issuing a leaf certificate") + } + + info, err := os.Stat(filepath.Join(dir, "client-render-07.key")) + if err != nil { + t.Fatalf("stat client key: %v", err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Errorf("client key mode = %04o, want 0600", got) + } +} + +func TestTLSIssue_ServerCertForRotation(t *testing.T) { + dir := existingFarm(t) + before := parseCertFile(t, filepath.Join(dir, "server.crt")) + + // Rotation replaces server.crt, so it needs --force. + if _, err := runTLSIssueCmd(t, "--out", dir, "--host", "sqi.example"); err == nil { + t.Fatal("overwrote server.crt without --force") + } + if _, err := runTLSIssueCmd(t, "--out", dir, "--host", "sqi.example", "--force"); err != nil { + t.Fatalf("tls issue --host --force: %v", err) + } + + after := parseCertFile(t, filepath.Join(dir, "server.crt")) + if after.SerialNumber.Cmp(before.SerialNumber) == 0 { + t.Error("server.crt was not reissued: same serial number") + } + ca := parseCertFile(t, filepath.Join(dir, "ca.crt")) + pool := x509.NewCertPool() + pool.AddCert(ca) + if _, err := after.Verify(x509.VerifyOptions{ + Roots: pool, DNSName: "sqi.example", + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("rotated server certificate does not verify: %v", err) + } +} + +func TestTLSIssue_RefusesToClobberAnExistingClientKey(t *testing.T) { + dir := existingFarm(t) + if _, err := runTLSIssueCmd(t, "--out", dir, "--client", "render-07"); err != nil { + t.Fatalf("first issue: %v", err) + } + original, err := os.ReadFile(filepath.Join(dir, "client-render-07.key")) + if err != nil { + t.Fatalf("read client key: %v", err) + } + + // A worker is already using this key; silently replacing it would take that + // worker offline at its next restart with no indication why. + _, err = runTLSIssueCmd(t, "--out", dir, "--client", "render-07") + if err == nil { + t.Fatal("reissued over an existing client key without --force") + } + if !strings.Contains(err.Error(), "--force") { + t.Errorf("error does not mention --force: %v", err) + } + + after, err := os.ReadFile(filepath.Join(dir, "client-render-07.key")) + if err != nil { + t.Fatalf("re-read client key: %v", err) + } + if !bytes.Equal(original, after) { + t.Error("client key changed despite the refusal") + } +} + +func TestTLSIssue_ErrorsWithoutACA(t *testing.T) { + dir := filepath.Join(t.TempDir(), "empty") + _, err := runTLSIssueCmd(t, "--out", dir, "--client", "w1") + if err == nil { + t.Fatal("issued against a directory with no CA") + } + if !strings.Contains(err.Error(), "tls init") { + t.Errorf("error does not point at `tls init`: %v", err) + } +} + +func TestTLSIssue_ErrorsWithNothingToIssue(t *testing.T) { + dir := existingFarm(t) + if _, err := runTLSIssueCmd(t, "--out", dir); err == nil { + t.Fatal("accepted an invocation with neither --client nor --host") + } +} + +func TestTLSInit_PointsAtIssueWhenTheCAExists(t *testing.T) { + dir := existingFarm(t) + // The original gap: this failed with no hint that another command does it. + _, err := runTLSInitCmd(t, "--out", dir, "--client", "render-07") + if err == nil { + t.Fatal("tls init overwrote an existing CA") + } + if !strings.Contains(err.Error(), "tls issue") { + t.Errorf("error does not point at `tls issue`: %v", err) + } +} diff --git a/cmd/sqi-worker/README.md b/cmd/sqi-worker/README.md index ed90ff8a..b06cf5b1 100644 --- a/cmd/sqi-worker/README.md +++ b/cmd/sqi-worker/README.md @@ -162,3 +162,4 @@ path), `--log-level`, and `--log-format`. `--dry-run` and - [`docs/worker-deployment.md`](../../docs/worker-deployment.md) — systemd, launchd, Windows service, Docker. - [`docs/worker-capabilities.md`](../../docs/worker-capabilities.md) — Capability tag reference. - [`docs/worker-docker.md`](../../docs/worker-docker.md) — Docker image quickstart. +- [`docs/tls.md`](../../docs/tls.md) — Connecting to a TLS broker and enrolling over HTTPS. diff --git a/cmd/sqi-worker/start.go b/cmd/sqi-worker/start.go index c334eed7..a9290e49 100644 --- a/cmd/sqi-worker/start.go +++ b/cmd/sqi-worker/start.go @@ -130,9 +130,9 @@ func runStart(cmd *cobra.Command, _ []string) error { // Resolve the NATS URL before dialing. If an explicit URL is configured // it is used as-is (mDNS bypassed). Otherwise mDNS is browsed // for "_sqi._tcp" services on the local network. If mDNS is - // disabled and no explicit URL is set, discovery.ResolveNATSURL returns a + // disabled and no explicit URL is set, discovery.Resolve returns a // clear error that is surfaced to the operator. - natsURL, err := workerdiscovery.ResolveNATSURL( + found, err := workerdiscovery.Resolve( ctx, cfg.NATS.URL, cfg.Discovery.EnableMDNS, @@ -145,7 +145,8 @@ func runStart(cmd *cobra.Command, _ []string) error { // Overwrite NATS.URL with the resolved address so natsclient.Connect and // all downstream log statements use the concrete URL. - cfg.NATS.URL = natsURL + cfg.NATS.URL = found.NATSURL + workerdiscovery.ApplyTLS(ctx, &cfg.NATS, found, logger) // ── Broker credential + NATS connection ────────────────────── // @@ -206,6 +207,11 @@ func runStart(cmd *cobra.Command, _ []string) error { logger, m, h, + obs.TLSConfig{ + Enabled: cfg.Metrics.TLS.Enabled, + CertFile: cfg.Metrics.TLS.CertFile, + KeyFile: cfg.Metrics.TLS.KeyFile, + }, ) go obsServer.Run(ctx) @@ -491,6 +497,10 @@ func connectToBroker( JoinToken: cfg.NATS.JoinToken, JoinTokenFile: cfg.NATS.JoinTokenFile, ServerURL: cfg.NATS.ServerURL, + // Without these the worker cannot enroll against an HTTPS server + // backed by a private farm CA, which is the first REST call it makes. + TLSCAFile: cfg.NATS.ServerTLSCAFile, + InsecureSkipVerify: cfg.NATS.ServerTLSInsecureSkipVerify, }, logger) noCredential := errors.Is(err, enroll.ErrNoCredential) if err != nil && !noCredential { diff --git a/config/sqi-server.example.yaml b/config/sqi-server.example.yaml index 03deb61a..b5d170ec 100644 --- a/config/sqi-server.example.yaml +++ b/config/sqi-server.example.yaml @@ -32,6 +32,24 @@ http: # Type: bool Env: SQI_HTTP_ENABLE_PPROF enable_pprof: false + # In-process TLS termination for the REST API, WebSocket gateway and web UI. + # + # OFF by default. When enabled the listener above UPGRADES IN PLACE — there + # is no plaintext port left, so enabling this on a running farm is a + # coordinated restart, not a rolling one. Generate a certificate pair with + # `sqi-server tls init`. Full guide: docs/tls.md. + # + # Certificates are validated at startup: a missing file, a mismatched key or + # an expired certificate is a startup error naming the key, and one expiring + # within 30 days logs a WARN. Rotation requires a restart. + # tls: + # # Type: bool Env: SQI_HTTP_TLS_ENABLED + # enabled: true + # # Type: string Env: SQI_HTTP_TLS_CERT_FILE + # cert_file: "/etc/sqi/certs/server.crt" + # # Type: string Env: SQI_HTTP_TLS_KEY_FILE + # key_file: "/etc/sqi/certs/server.key" + # ── NATS JetStream (embedded broker) ───────────────────────────────────────── nats: # TCP address the embedded NATS server binds to. @@ -59,6 +77,30 @@ nats: # Broker authentication for worker connections. Opt-in and independent of # the top-level auth block (auth.enabled) below — that block gates human # users and API keys; this one gates workers connecting to the broker. + # In-process TLS on the embedded broker's client listener. + # + # OFF by default. This is what encrypts the WORKER TRANSPORT — assignments + # (command lines, embedded file contents, parameters, environment, the + # isolation username), status messages and log chunks. Broker AUTHENTICATION + # below does not encrypt anything: the two are independent settings against + # different threats, and a production farm wants both. + # + # Workers need the matching CA in nats.tls_ca_file. See docs/tls.md. + # tls: + # # Type: bool Env: SQI_NATS_TLS_ENABLED + # enabled: true + # # Type: string Env: SQI_NATS_TLS_CERT_FILE + # cert_file: "/etc/sqi/certs/server.crt" + # # Type: string Env: SQI_NATS_TLS_KEY_FILE + # key_file: "/etc/sqi/certs/server.key" + # # Require each worker to present a client certificate signed by this CA + # # (mutual TLS). Layers ON TOP of the nkey credential below rather than + # # replacing it: the certificate gates who may connect, the nkey decides + # # which worker they are. Issue per-worker certificates with + # # `sqi-server tls init --client `. + # # Type: string Env: SQI_NATS_TLS_CLIENT_CA_FILE + # client_ca_file: "/etc/sqi/certs/ca.crt" + auth: # Requires every NATS client to present a per-worker nkey credential. # When false, the broker accepts any connection (the v0.3.0 behavior). diff --git a/config/sqi-worker.example.yaml b/config/sqi-worker.example.yaml index 9f42e84f..2761425f 100644 --- a/config/sqi-worker.example.yaml +++ b/config/sqi-worker.example.yaml @@ -31,8 +31,23 @@ nats: # Type: string Env: SQI_WORKER_NATS_URL url: "" + # Whether to use TLS on the broker connection. THREE-VALUED, not a bool: + # + # "auto" (default) use TLS when there is a reason to: any tls_* field + # below is set, OR the server found over mDNS advertises + # a TLS-required broker. + # "true" always use TLS, even with no CA configured (system + # roots) — for a publicly-trusted broker certificate. + # "false" never use TLS, even with a CA configured. + # + # Any other value is rejected at startup. See docs/tls.md. + # Type: string Env: SQI_WORKER_NATS_TLS_ENABLED + tls_enabled: "auto" + # Path to the client TLS certificate (PEM-encoded). Leave empty to disable - # mutual TLS. Required only when the server demands client certificates. + # mutual TLS. Required only when the server demands client certificates + # (nats.tls.client_ca_file on the server). Issue one per worker with + # `sqi-server tls init --client `. # Type: string Env: SQI_WORKER_NATS_TLS_CERT_FILE tls_cert_file: "" @@ -41,11 +56,28 @@ nats: # Type: string Env: SQI_WORKER_NATS_TLS_KEY_FILE tls_key_file: "" - # Path to the CA certificate used to verify the server's TLS certificate - # (PEM-encoded). Leave empty to use the system CA pool. + # Path to the CA certificate used to verify the BROKER's TLS certificate + # (PEM-encoded). Empty means the system CA pool; set means THAT CA only, so + # a farm CA pins the acceptable issuer rather than widening it. # Type: string Env: SQI_WORKER_NATS_TLS_CA_FILE tls_ca_file: "" + # Path to the CA certificate used to verify sqi-server's HTTPS certificate at + # nats.server_url, for enrollment. Separate from tls_ca_file above because + # the two verify different certificates on different ports, which may come + # from different issuers. On a farm generated by `sqi-server tls init` they + # are simply the same ca.crt. + # + # Enrollment runs over REST BEFORE the worker holds any broker credential, + # so without this a farm with an HTTPS server cannot bootstrap at all. + # Type: string Env: SQI_WORKER_NATS_SERVER_TLS_CA_FILE + server_tls_ca_file: "" + + # Skip verification of sqi-server's HTTPS certificate during enrollment. + # Development environments only. + # Type: bool Env: SQI_WORKER_NATS_SERVER_TLS_INSECURE_SKIP_VERIFY + server_tls_insecure_skip_verify: false + # Skip TLS certificate verification. For development environments only. # Never set this to true in production — it defeats the purpose of TLS. # Type: bool Env: SQI_WORKER_NATS_INSECURE_SKIP_VERIFY @@ -257,10 +289,27 @@ metrics: # TCP address the local HTTP server listens on. # Change only if port 9091 is in use on this host. Use "0.0.0.0:9091" to # expose the endpoints to Prometheus scrapers on the network (ensure the port - # is firewalled appropriately). + # is firewalled appropriately, and turn on tls below). # Type: string Env: SQI_WORKER_METRICS_ADDR addr: "127.0.0.1:9091" + # Terminate TLS on THIS listener — the worker's own metrics/health endpoint, + # not its connection to the server (that is nats.tls_* above). + # + # Off by default, which is right for the loopback address above. Turn it on + # together with any addr reachable from elsewhere: this endpoint serves + # metrics, health and, with enable_pprof, full runtime profiles. + # + # The worker needs a certificate of its own here; issue one with + # `sqi-server tls issue --host `. See docs/tls.md. + # tls: + # # Type: bool Env: SQI_WORKER_METRICS_TLS_ENABLED + # enabled: true + # # Type: string Env: SQI_WORKER_METRICS_TLS_CERT_FILE + # cert_file: "/etc/sqi/certs/worker.crt" + # # Type: string Env: SQI_WORKER_METRICS_TLS_KEY_FILE + # key_file: "/etc/sqi/certs/worker.key" + # Expose Go runtime profiling endpoints at /debug/pprof/. # Profiling data reveals memory layout, goroutine stacks, and CPU hotspots — # enable temporarily for performance diagnosis, never in long-term production. diff --git a/deploy/README.md b/deploy/README.md index 69dd17be..9d129f2d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -8,6 +8,22 @@ Deployment artifacts for `sqi`: - `deploy/systemd/` — sample unit files for bare-metal Linux worker installs. - `deploy/config/` — pointer to the canonical example configs under `config/`. +## TLS + +Every artifact here deploys sqi in its **plaintext default**: the REST API, the +WebSocket gateway and the embedded broker all serve unencrypted. That is +deliberate — TLS is opt-in — but it means none of these files is production-ready +as shipped on an untrusted network. + +To add TLS, generate material with `sqi-server tls init`, mount it into the +container or place it on the host, and set `http.tls` and `nats.tls` (plus the +matching worker CA settings). The full guide, including the fact that enabling +TLS on a running farm is a coordinated restart rather than a rolling one, is in +[`docs/tls.md`](../docs/tls.md). + +Terminating TLS at a reverse proxy in front of the API is equally supported — +but note it does nothing for the broker, which workers connect to directly. + Kubernetes manifests / a Helm chart for production-mode deployments are planned for a later phase and are not shipped yet. diff --git a/deploy/config/README.md b/deploy/config/README.md index d9d2381b..16b31ce7 100644 --- a/deploy/config/README.md +++ b/deploy/config/README.md @@ -9,3 +9,6 @@ under [`config/`](../../config/): Copy the relevant file to one of the search locations (`./config/`, `~/.sqi/`, `/etc/sqi/`) and edit. They are kept at the repo root so the dev workflow and the deployment docs reference one source of truth. + +Both files ship with TLS commented out, matching the plaintext default. See +[`docs/tls.md`](../../docs/tls.md) for what to uncomment and in what order. diff --git a/docs/README.md b/docs/README.md index 5c51f4a6..04880c64 100644 --- a/docs/README.md +++ b/docs/README.md @@ -33,6 +33,7 @@ Long-form documentation for `sqi-server` operators and contributors. - `configuration.md` — every server config option with type, default, env var, example - `auth.md` — the opt-in `auth.enabled` gate, the `Principal`/`Authenticator` model, and what's scaffolding vs. live +- `tls.md` — in-process TLS for the API and the embedded broker, `sqi-server tls init`, and optional worker mTLS - `worker-configuration.md` — the full `sqi-worker` config reference - `worker-capabilities.md` — capability tags and software auto-detection - `worker-deployment.md` — running workers (systemd, bare binary) diff --git a/docs/architecture.md b/docs/architecture.md index 384182b8..de117ac2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -645,6 +645,16 @@ who published a message it received (`bus.ParseWorkerSubject`). A queue-unaffiliated worker leases on the reserved queue token `work.lease.._any` (`bus.WildcardQueueToken`). +**Transport security.** Every subject above crosses the broker's client +listener, which is **plaintext unless `nats.tls` is configured** — see +[`docs/tls.md`](tls.md). That matters most for `work.lease.*`, whose reply +carries the whole `AssignMsg`: command lines, embedded file contents, job +parameters, environment and the run-as-user name. Broker *authentication* +(`nats.auth`, [`docs/auth.md`](auth.md)) is a separate setting: it decides who +may connect and constrains which subjects they may use, but it does not encrypt +anything. The server's own connections to the embedded broker never touch this +listener at all — they run in-process over a pipe. + JetStream streams use file-backed storage with configurable size limits. `work.lease..` uses core NATS request/reply — no stream is created for it. The server holds an unfulfillable request in memory for up to 30 s before diff --git a/docs/auth.md b/docs/auth.md index 9f96f22e..8010bdd7 100644 --- a/docs/auth.md +++ b/docs/auth.md @@ -39,6 +39,11 @@ worker; it does not stop someone who can already read the network from reading what crosses it. Do not conclude "I turned on broker authentication" means "the channel is private" — it does not. +Encrypting that transport is a separate, independent setting: `nats.tls`. See +[`docs/tls.md`](tls.md). A production farm wants both — authentication decides +*who* may connect, TLS decides *who may read what crosses the wire* — and +neither substitutes for the other. + ### The credential: an nkey per worker Each worker holds an Ed25519 **nkey** keypair. The worker generates it @@ -466,9 +471,13 @@ A queue may set `run_as_user` (and optionally `run_as_group`) via attaches that **username only** — never a credential — to every task assignment for that queue's tasks (`AssignMsg.Isolation` / `protocol.IsolationSpec`, `internal/worker/protocol/protocol.go`). This is -deliberate: worker↔server transport has no authentication at all today (see -[Known gaps](#known-gaps)), so nothing secret can be allowed to travel on -that channel. The worker resolves the username to a real OS credential +deliberate: both worker↔server transport protections are **opt-in and off by +default** — authentication via `nats.auth` (see +[Broker authentication (transport)](#broker-authentication-transport)) and +encryption via `nats.tls` ([`docs/tls.md`](tls.md)) — so the assignment +channel must be assumed readable, and nothing secret can be allowed to travel +on it. That holds regardless of what any individual deployment turns on: the +protocol is designed for the weakest configuration it supports. The worker resolves the username to a real OS credential locally and runs both job-code launch sites under it — an OpenJD environment's `onEnter`/`onExit` actions and a task's own actions — with a filtered environment and a session working directory private to that user. @@ -637,7 +646,7 @@ The cookie: `cookie_secure` is a **3-valued string, not a bool**, because sqi's default deployment posture is a trusted, plain-HTTP LAN (`http.addr` defaults to -`0.0.0.0:8080`, no TLS). `"auto"` sets `Secure` when the request arrived over +`0.0.0.0:8080`, and TLS is opt-in — see [`docs/tls.md`](tls.md)). `"auto"` sets `Secure` when the request arrived over TLS or carries `X-Forwarded-Proto: https`, which is right behind a TLS-terminating proxy that sets that header — but an operator on plain HTTP, with no such proxy, needs to be able to force `Secure` off explicitly rather @@ -783,6 +792,17 @@ enabled — letting an attacker-registrable origin (like `https://app.example.com.evil.io`) ride a victim's session cookie. Name every allowed origin explicitly. +**Origins are scheme-sensitive, so enabling TLS invalidates `http://` +entries.** The Origin check compares scheme, host and port — `http://ui.example.com` +and `https://ui.example.com` are different origins, and matching on host alone +would accept a plaintext origin against a TLS deployment. If you turn on +`http.tls` ([`docs/tls.md`](tls.md)), or move behind a TLS-terminating proxy, +update every entry here to `https://` at the same time or credentialed +cross-origin requests from the UI will start returning `403`. The same applies +to the same-origin path: it resolves the request's scheme from `r.TLS` or +`X-Forwarded-Proto`, so a proxy that does not set that header makes a genuinely +same-origin `https://` request look cross-origin. + Leaving the list empty keeps the previous default of `["*"]`. **The wildcard-drop above still applies**: with auth enabled, `"*"` — whether explicit or defaulted — is dropped and credentialed cross-origin requests @@ -1710,7 +1730,9 @@ provider](development.md#testing-against-a-real-directory-or-identity-provider) attackers, and any host that can reach the embedded NATS broker's port (`4222` by default) can register as a worker and receive task assignments. Turning broker authentication on does not encrypt the transport either — - task payloads, assignments and log chunks stay cleartext regardless. + task payloads, assignments and log chunks stay cleartext unless `nats.tls` + is also configured ([`docs/tls.md`](tls.md)); the two settings are + independent and address different threats. `sqi-server` emits a startup WARN when the broker address is non-loopback and broker auth is off, precisely because an operator reading "I flipped `auth.enabled` to `true`, so the server is now locked down" should not diff --git a/docs/configuration.md b/docs/configuration.md index 1afe872c..6aa01a46 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -123,6 +123,66 @@ SQI_HTTP_CORS_ORIGINS="https://ui.example.com,http://localhost:5173" sqi-server serve --http-cors-origins=https://ui.example.com ``` +### `http.tls.enabled` + +| | | +|---|---| +| **Type** | `bool` | +| **Default** | `false` | +| **Env var** | `SQI_HTTP_TLS_ENABLED` | + +Terminate TLS in-process on the REST/WebSocket listener. **The listener +upgrades in place — there is no plaintext port when this is on.** A plaintext +request to the TLS port receives a `400` rather than hanging. + +Requires [`http.tls.cert_file`](#httptlscert_file) and +[`http.tls.key_file`](#httptlskey_file); the server refuses to start without +both. See [`docs/tls.md`](tls.md) for the full model, including the +coordinated restart that enabling this requires on a running farm. + +```yaml +http: + tls: + enabled: true + cert_file: "/etc/sqi/certs/server.crt" + key_file: "/etc/sqi/certs/server.key" +``` + +--- + +### `http.tls.cert_file` + +| | | +|---|---| +| **Type** | `string` | +| **Default** | `""` | +| **Env var** | `SQI_HTTP_TLS_CERT_FILE` | + +PEM certificate served on the HTTP listener: leaf first, then any +intermediates. Generate a pair with `sqi-server tls init`. + +Validated at startup — a missing file, a key that does not match, or an +already-expired certificate is a startup error naming this key, never a +runtime surprise. A certificate expiring within 30 days logs a WARN. Setting +this while `http.tls.enabled` is `false` logs a WARN that the server is +serving plaintext. + +--- + +### `http.tls.key_file` + +| | | +|---|---| +| **Type** | `string` | +| **Default** | `""` | +| **Env var** | `SQI_HTTP_TLS_KEY_FILE` | + +PEM private key matching [`http.tls.cert_file`](#httptlscert_file). Should be +mode `0600`; `sqi-server tls init` writes it that way. + +Certificates are read once at startup. **Rotation requires a restart** — +replacing the files under a running server has no effect until it restarts. + --- ## `nats` — Embedded NATS JetStream broker @@ -276,6 +336,89 @@ nats: --- +### `nats.tls.enabled` + +| | | +|---|---| +| **Type** | `bool` | +| **Default** | `false` | +| **Env var** | `SQI_NATS_TLS_ENABLED` | + +Require TLS on every connection to the embedded broker. + +This is what encrypts the **worker transport** — task assignments (command +lines, embedded file contents, parameters, environment, the isolation +username), status messages and log chunks. Broker *authentication* +([`nats.auth.enabled`](#natsauthenabled)) does not encrypt anything; the two +settings are independent and address different threats. A production farm +wants both. + +Requires [`nats.tls.cert_file`](#natstlscert_file) and +[`nats.tls.key_file`](#natstlskey_file). Workers must be configured with the +matching CA (`nats.tls_ca_file`) — see [`docs/tls.md`](tls.md). + +```yaml +nats: + tls: + enabled: true + cert_file: "/etc/sqi/certs/server.crt" + key_file: "/etc/sqi/certs/server.key" +``` + +--- + +### `nats.tls.cert_file` + +| | | +|---|---| +| **Type** | `string` | +| **Default** | `""` | +| **Env var** | `SQI_NATS_TLS_CERT_FILE` | + +PEM certificate the broker presents to connecting workers. May be the same +file as [`http.tls.cert_file`](#httptlscert_file), or a different one — the +two listeners are configured independently, so an operator can serve the API +with a publicly-issued certificate and the broker with a private farm CA. + +Validated at startup on the same terms as `http.tls.cert_file`. + +--- + +### `nats.tls.key_file` + +| | | +|---|---| +| **Type** | `string` | +| **Default** | `""` | +| **Env var** | `SQI_NATS_TLS_KEY_FILE` | + +PEM private key matching [`nats.tls.cert_file`](#natstlscert_file). + +--- + +### `nats.tls.client_ca_file` + +| | | +|---|---| +| **Type** | `string` | +| **Default** | `""` | +| **Env var** | `SQI_NATS_TLS_CLIENT_CA_FILE` | + +When set, every connecting worker must present a client certificate signed by +this CA (mutual TLS). Issue per-worker certificates with +`sqi-server tls init --client `. + +This **layers on** the per-worker nkey credential rather than replacing it: the +certificate gates who may open a connection, the nkey decides which worker they +are and drives subject permissions and revocation. A valid farm certificate on +its own authenticates nobody. sqi does not bind a certificate to a worker ID. + +Requires [`nats.tls.enabled`](#natstlsenabled) — setting this on a plaintext +broker is a startup error, since client certificates cannot be verified where +there is no TLS handshake. + +--- + ## `store` — SQLite state store ### `store.sqlite_path` diff --git a/docs/development.md b/docs/development.md index bbd8a789..d06bdd5c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -81,6 +81,7 @@ Run `make` (no arguments) to see all available targets with descriptions. | `make test-oidc` | Run the SSO tests against a real Keycloak in a container (needs Docker; **skips** without it) | | `make test-isolation` | Run run-as-user task-isolation tests as real root against real OS accounts in a container (needs Docker; **skips** without it) | | `make test-isolation-windows` | Run the Windows run-as-user isolation tests against real local accounts — must be run from an **elevated** shell on a real Windows host (no container); exits 0 with a message when not elevated | +| `make test-discovery` | Run the mDNS discovery tests over **real multicast** (no container; **fails rather than skips** when multicast is unavailable) | | `make test-conformance` | Run the official OpenJD conformance suite against the vendored `third_party/` fixtures (build tag `conformance`) | | `make test-expr-oracle` | Differential-test the EXPR evaluator against the OpenJD reference implementation (needs `python3`; **skips** without it) | | `make test-preset-library` | Validate the **published** preset library against the validator in your tree (needs network; **skips** when the library is unreachable, **fails** when it is reachable but invalid) | @@ -109,6 +110,72 @@ Override the coverage threshold: `make test-cover COVERAGE_MIN=50` --- + +## Testing mDNS discovery over real multicast + +`make test-discovery` runs the discovery suite against a **real** mDNS round +trip: a real responder advertises on a real interface and a real browser finds +it. It needs no container — unlike LDAP, SSO and isolation, nothing here is +unavailable natively. + +What it covers that unit tests cannot: the server decides its TXT records from +its own config, they cross the wire, and the worker parses them back and acts on +them. The halves were each unit-tested for a while with nothing joining them, +which is how `nats_tls` came to be advertised by the server and read by nobody. +`TestDiscovery_RealBinaryFindsItsServerOverMDNS` goes furthest: a real +`sqi-worker` subprocess with **no** `nats.url` at all has to find its server, +learn from the advertisement that the broker needs TLS, and register. + +### It does not advertise on your network + +Every advertisement these tests make is restricted to **loopback**. A browser +listening on all interfaces still receives it — that is verified rather than +assumed — so the coverage is unaffected, and a test run never announces a +service on the LAN it happens to be attached to. This is an invariant, not a +preference: where loopback cannot carry multicast the tests refuse to run rather +than quietly falling back to a real interface. + +Linux loopback needs two things macOS lo0 has by default, and the tests state +each one when it is missing: + +```bash +sudo ip link set lo multicast on # Linux `lo` ships without the flag +sudo ip -6 addr add fe80::1/64 dev lo # and with no non-loopback address +``` + +The second is the less obvious one. zeroconf builds the advertisement's address +records from the advertising interface's own addresses and **discards loopback +ones**, so on Linux — whose `lo` carries only `127.0.0.1` and `::1` — there is +nothing left to advertise and registration fails outright with "Could not +determine host IP addresses", multicast flag or no multicast flag. macOS lo0 +also carries `fe80::1`, which is the whole reason this is invisible on a Mac. +Adding that same link-local address makes the two hosts behave alike; it is +scoped to the link and is not routed anywhere. + +The CI job runs both. + +### The one test that does open a listener + +`TestDiscovery_RealBinaryFindsItsServerOverMDNS` binds the test broker to all +interfaces for about ten seconds, so it runs **only** under `make +test-discovery` — `make test-integration` skips it. It cannot avoid the +listener: the mDNS advertisement carries this machine's *hostname*, so the +worker dials that name whatever interface the announcement went out on, and a +loopback-bound broker is unreachable there. Broker authentication and TLS are +both on, so enrolling still requires a join token. + +### Two more things + +- **It refuses to run next to a real farm.** The tests never advertise beyond + loopback, but they still *browse* on every interface, because the production + worker does. If anything else is already advertising `_sqi._tcp` they skip, + rather than risk discovering a colleague's server — the test cannot tell their + production broker from its own before connecting to it. +- **A skip is a failure here.** `SQI_TEST_REQUIRE_MULTICAST=1` (which the target + sets) turns the capability skip into a failure, so the target cannot pass + while running nothing. The same tests are part of `make test-integration`, + where they skip cleanly on a host without loopback multicast. + ## Running tests ```sh diff --git a/docs/operations.md b/docs/operations.md index 6763c6b7..4fa4aa8d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -80,6 +80,10 @@ Minimum required changes for production: - `store.sqlite_path` — set to an absolute path on a local SSD - `nats.data_dir` — set to a persistent directory for JetStream storage - `http.addr` — restrict to `127.0.0.1:8080` if a reverse proxy handles TLS +- `http.tls` / `nats.tls` — terminate TLS in sqi instead, if there is no proxy. + Both are off by default, so **the API and the worker transport are plaintext + until you set one or the other**. See [`docs/tls.md`](tls.md); note that + enabling TLS on a running farm is a coordinated restart, not a rolling one See [`docs/configuration.md`](configuration.md) for all options. diff --git a/docs/tls.md b/docs/tls.md new file mode 100644 index 00000000..1d74ca55 --- /dev/null +++ b/docs/tls.md @@ -0,0 +1,307 @@ +# TLS + +sqi can terminate TLS in-process on every listener it opens: `sqi-server`'s REST +API, WebSocket gateway and embedded web UI on one listener, its embedded NATS +broker on another, and the worker's own metrics and health endpoint on a third. +Workers can optionally be required to present a client certificate. + +**TLS is off by default.** A `sqi-server` started with no TLS configuration +behaves exactly as it did before TLS existed: one plaintext HTTP listener, one +plaintext broker. Everything below is opt-in. + +Running behind a TLS-terminating reverse proxy is equally supported and is what +many deployments will do anyway — see [Reverse proxy](#reverse-proxy). + +## What is encrypted, and what is not + +| Surface | Covered by | Notes | +|---|---|---| +| REST API, WebSocket, web UI | `http.tls` | One listener; it upgrades in place | +| Worker ↔ broker (assignments, status, logs) | `nats.tls` | Closes the cleartext gap that broker authentication alone does not | +| Worker enrollment (`POST /api/v1/workers/enroll`) | `http.tls` server-side, `nats.server_tls_ca_file` worker-side | Runs over REST before the worker holds any credential | +| Outbound LDAP / OIDC | Their own settings (`auth.ldap.*`, `auth.oidc.*`) | Unchanged by anything here | +| Worker metrics / health (`metrics.addr`) | `metrics.tls` (worker) | Loopback by default, where plaintext is fine; turn TLS on with any address reachable from elsewhere, since this carries metrics, health and — with `enable_pprof` — full runtime profiles | + +Two things TLS does **not** do: + +- **It is not authentication.** Broker TLS encrypts the worker transport; it does + not decide *which* worker is connecting. That remains the per-worker nkey + credential — see [`docs/auth.md`](auth.md). The two are independent and a + production farm wants both. +- **It does not protect data at rest.** The SQLite database, JetStream's + file-backed streams and worker session directories are unaffected. + +## Configuration + +### Server + +```yaml +http: + tls: + enabled: false + cert_file: "" # PEM certificate: leaf first, then any intermediates + key_file: "" # PEM private key matching cert_file + +nats: + tls: + enabled: false + cert_file: "" + key_file: "" + client_ca_file: "" # set = require and verify worker client certificates +``` + +Environment: `SQI_HTTP_TLS_ENABLED`, `SQI_HTTP_TLS_CERT_FILE`, +`SQI_HTTP_TLS_KEY_FILE`, `SQI_NATS_TLS_ENABLED`, `SQI_NATS_TLS_CERT_FILE`, +`SQI_NATS_TLS_KEY_FILE`, `SQI_NATS_TLS_CLIENT_CA_FILE`. + +The two blocks are deliberately independent. An operator may legitimately serve +the API with a publicly-issued certificate and the broker with a private farm CA, +or turn on one without the other. + +There is no minimum-version or cipher-suite setting. Go's server default is +already TLS 1.2 minimum, and a knob there is a footgun with no farm use case. + +### Worker + +```yaml +nats: + url: "nats://sqi-server.example:4222" + tls_enabled: "auto" # "auto" | "true" | "false" + tls_ca_file: "" # CA that verifies the BROKER certificate + tls_cert_file: "" # client certificate, for mTLS + tls_key_file: "" + server_url: "https://sqi-server.example:8080" + server_tls_ca_file: "" # CA that verifies the API certificate + server_tls_insecure_skip_verify: false + +metrics: + addr: "127.0.0.1:9091" + tls: # the worker's own metrics listener + enabled: false + cert_file: "" + key_file: "" +``` + +Environment: `SQI_WORKER_NATS_TLS_ENABLED`, `SQI_WORKER_NATS_TLS_CA_FILE`, +`SQI_WORKER_NATS_TLS_CERT_FILE`, `SQI_WORKER_NATS_TLS_KEY_FILE`, +`SQI_WORKER_NATS_SERVER_TLS_CA_FILE`, +`SQI_WORKER_NATS_SERVER_TLS_INSECURE_SKIP_VERIFY`, +`SQI_WORKER_METRICS_TLS_ENABLED`, `SQI_WORKER_METRICS_TLS_CERT_FILE`, +`SQI_WORKER_METRICS_TLS_KEY_FILE`. + +`metrics.tls` is independent of everything else here: it protects the worker's +own listener, not its connection to the server. A worker needs a certificate of +its own for it — issue one with `sqi-server tls issue --host `, or +use whatever your monitoring stack already trusts. + +`tls_ca_file` and `server_tls_ca_file` are separate keys on purpose: they verify +two different certificates, on two different ports, which can legitimately come +from different issuers. On a farm generated by `sqi-server tls init` they are +simply the same `ca.crt`. + +**`tls_enabled` is a three-valued string, not a bool** — the same shape as the +server's `auth.session.cookie_secure`: + +| Value | Meaning | +|---|---| +| `"auto"` (default) | Use TLS when there is a reason to: any `tls_*` field is set, **or** the discovered server advertises a TLS-required broker | +| `"true"` | Always use TLS, even with no CA configured (system roots) — for a broker presenting a publicly-trusted certificate, which nothing else would signal | +| `"false"` | Never use TLS, even with a CA configured — an explicit override for an operator who knows the broker is plaintext | + +A bool cannot express this: the interesting states are *infer* and *force off*, +and a bool collapses both onto its zero value. The usual boolean spellings +(`1`/`yes`/`on`, `0`/`no`/`off`) are accepted as synonyms in both the file and +the environment; anything else is rejected at startup rather than silently +falling back to `auto`. + +### Discovery fills in "auto" + +A worker that finds its server over mDNS reads the `nats_tls` TXT record the +server advertises. Under `"auto"` that record is a reason to use TLS, so a +discovered TLS farm connects over TLS with nothing configured on the worker at +all. Two consequences worth knowing: + +- With no `tls_ca_file`, the broker certificate is verified against the **system + roots**, so a farm CA fails with a certificate error naming the issuer. The + worker warns about exactly this at startup. Copy `ca.crt` over and set + `tls_ca_file`. +- Under `"false"` the advertisement is **not** overridden — false means false — + but the worker warns that the connection is about to be refused, because the + broker's own error will not mention the setting responsible. + +An explicit `nats.url` discovers nothing, so none of this applies: an operator +who configures the URL by hand configures the transport by hand. + +### CA semantics + +Uniform in every component: **an empty CA setting means the system roots; a CA +setting means that CA only.** Pointing a worker at a farm CA *pins* the acceptable +issuer rather than adding to the public set. A deployment using publicly-issued +certificates leaves the CA settings empty. + +### Validation + +Certificate problems are refused **at load**, with a message naming the config key. +Nothing about a certificate is allowed to fail for the first time at connect time. + +| Condition | Result | +|---|---| +| `enabled: true` with an empty `cert_file` or `key_file` | error at startup | +| File missing, unreadable, or key/certificate mismatch | error at startup | +| Certificate already expired | error at startup | +| Certificate expires within 30 days | WARN at startup | +| Certificate files set while `enabled: false` | WARN — "serving plaintext" | +| `nats.tls.client_ca_file` set while `nats.tls.enabled: false` | error at startup | + +An expired certificate is an error rather than a warning because a server that +boots with one is a server whose entire farm fails to connect — strictly worse +than refusing to boot with an actionable message. + +## Generating certificates + +```bash +sqi-server tls init --out ./certs --host sqi-server.example +``` + +Writes a farm CA (`ca.crt`, `ca.key` — ECDSA P-256, 10 years) and a server +certificate (`server.crt`, `server.key` — 2 years) covering the hosts given by +`--host`, plus `localhost`, `127.0.0.1` and `::1` in every case. Private keys are +written `0600`; certificates `0644`. On success it prints the exact configuration +keys to set on each side. + +`--host` is repeatable. Loopback is always included even when `--host` names +specific hosts: a certificate naming only the LAN host cannot be verified from the +machine itself, which breaks local health probes and any client reaching the server +by the name it actually runs under. + +For the optional mTLS path, issue per-worker client certificates: + +```bash +sqi-server tls init --out ./certs --host sqi-server.example --client render-01 --client render-02 +``` + +The command **refuses to overwrite an existing `ca.key`**. Replacing a farm CA +invalidates every certificate ever issued from it, so that has to be a deliberate +act: move the old directory aside first. + +That refusal means `tls init` cannot be re-run to add a worker later — use +`tls issue`. + +### Adding a worker, or rotating the server certificate + +```bash +# add a worker to a farm that already exists +sqi-server tls issue --out ./certs --client render-07 + +# rotate the server certificate (replaces server.crt, so --force) +sqi-server tls issue --out ./certs --host sqi-server.example --force +``` + +`tls issue` signs from the CA already in `--out` and never touches it, so every +certificate already deployed keeps working. Distribute the new +`client-.crt`/`.key` to that worker and set `nats.tls_cert_file` / +`nats.tls_key_file`; no other worker needs to change. + +It **refuses to replace an existing file without `--force`**. A client key that +is already deployed belongs to a running worker, and replacing it takes that +worker offline at its next restart with nothing to indicate why. + +A rotated server certificate is only picked up on restart — certificates are +read once, at startup. + +The command exists because SANs and extended key usage are exactly what +hand-rolled `openssl` invocations get wrong, and getting them wrong fails at +worker-connect time — far from the mistake — rather than at load. Any CA works; +this one just makes the common case correct by default. + +## Worker client certificates (mTLS) + +Setting `nats.tls.client_ca_file` makes the broker require a client certificate +signed by that CA from every connecting worker. + +This **layers on** the nkey credential rather than replacing it. The certificate +gates *who may open a connection*; the nkey decides *which worker* they are, and +remains what drives subject permissions, per-worker inbox prefixes and revocation. +A valid farm certificate on its own authenticates nobody. + +sqi does not bind a client certificate to a worker ID: any enrolled nkey may be +presented over any valid farm client certificate. If you need certificate-to-worker +binding, that is not available today. + +The server's own connection to its own broker is exempt from the client-certificate +requirement. It runs in-process over a pipe rather than the network, and requiring +a certificate of the process that already holds the broker's private key would be +meaningless. Network peers are never exempt. + +## Enabling TLS on a running farm + +**This is a coordinated restart, not a rolling one.** When TLS is enabled the +listener upgrades in place — there is no plaintext port left — and a worker +configured for TLS cannot talk to a plaintext broker. There is no window in which +both old and new workers can connect. + +The supported order: + +1. `sqi-server tls init --out ./certs --host ` +2. Copy `ca.crt` to every worker. +3. Stage each worker's `tls_ca_file` and `server_tls_ca_file` in its config, but + do not restart the workers yet. +4. Set `http.tls` and `nats.tls` on the server, then restart the server and all + workers together. + +## Certificate rotation + +Rotation requires a **server restart**. Certificates are read once, at startup, +and validated before the listener opens; there is no hot-reload. Replacing the +files under a running server has no effect until it restarts. + +Renew before expiry: the server refuses to start with an expired certificate, and +warns for 30 days beforehand. + +## Reverse proxy + +Terminating TLS at nginx, Caddy, Traefik or a cloud load balancer is fully +supported and needs no TLS configuration in sqi. Two things to get right: + +- **Set `X-Forwarded-Proto: https`.** Session cookies default to + `auth.session.cookie_secure: "auto"`, which resolves `Secure` from the request's + TLS state or that header. Without it, cookies issued behind a proxy are not + marked `Secure`. The CSRF origin check reads the same signal. +- **The link from the proxy to sqi is still plaintext** unless you also enable + `http.tls` and point the proxy at HTTPS. On a single host over loopback that is + usually fine; across a network it is not. + +A reverse proxy in front of the **API** does nothing for the **broker**. Workers +connect to the NATS port directly, so `nats.tls` is the only thing that encrypts +assignments, status and log chunks. + +## Verifying + +```bash +curl --cacert ./certs/ca.crt https://sqi-server.example:8080/readyz +``` + +The startup log states the transport plainly: + +``` +level=INFO msg="http: listening" addr=0.0.0.0:8080 tls=true url=https://localhost:8080 +``` + +A worker that fails to connect over TLS reports the reason rather than retrying +silently: a certificate signed by an unknown CA fails verification with that named +as the cause, and a worker with no TLS configuration attempting a TLS-required +broker fails with `secure connection not available`. + +The server advertises `tls=1` / `nats_tls=1` in its mDNS TXT records when the +respective listener is TLS-terminated, and a discovering worker acts on +`nats_tls` — see [Discovery fills in "auto"](#discovery-fills-in-auto). + +`scripts/smoke.sh` runs its whole flow in three modes — plain, broker-auth and TLS +— so the TLS path is exercised end to end on every CI run. + +## See also + +- [`docs/auth.md`](auth.md) — the authentication model, including broker + authentication and worker enrollment +- [`docs/configuration.md`](configuration.md) — every configuration key +- [`docs/architecture.md`](architecture.md) — NATS subjects and data flow diff --git a/docs/worker-configuration.md b/docs/worker-configuration.md index 79b1767a..c6a0849e 100644 --- a/docs/worker-configuration.md +++ b/docs/worker-configuration.md @@ -89,6 +89,46 @@ Path to the client TLS private key (PEM-encoded). Must be set together with --- +### `nats.tls_enabled` + +| | | +|---|---| +| **Type** | `string` — `"auto"`, `"true"` or `"false"` | +| **Default** | `"auto"` | +| **Env var** | `SQI_WORKER_NATS_TLS_ENABLED` | + +Whether to use TLS on the broker connection. Three-valued, mirroring the +server's [`auth.session.cookie_secure`](configuration.md): + +| Value | Meaning | +|---|---| +| `"auto"` | Use TLS when there is a reason to: any `tls_*` field is set, **or** the server discovered over mDNS advertises a TLS-required broker (`nats_tls=1`) | +| `"true"` | Always use TLS, even with no CA configured — verification then uses the system roots, which is right for a publicly-trusted broker certificate and wrong for a farm CA | +| `"false"` | Never use TLS, even with a CA configured | + +A bool cannot express this: the interesting states are *infer* and *force off*, +and a bool collapses both onto its zero value. + +The usual boolean spellings are accepted as synonyms in both the config file and +the environment — `1`/`yes`/`on` for `"true"`, `0`/`no`/`off` for `"false"` — +since `SQI_WORKER_NATS_TLS_ENABLED=1` is how boolean-looking variables get set. +Anything else is rejected at startup with a `nats.tls_enabled` validation error, +rather than silently falling back to `"auto"`. + +Under `"auto"`, a worker that discovers a TLS-required broker enables TLS and +says so in its startup log; if no `tls_ca_file` is configured it also warns that +the system roots cannot verify a farm CA. Under `"false"` the advertisement is +honored rather than overridden, but the worker warns that the connection is +about to be refused — the broker's own error names neither the cause nor this +setting. + +```yaml +nats: + tls_enabled: "auto" +``` + +--- + ### `nats.tls_ca_file` | | | @@ -97,8 +137,15 @@ Path to the client TLS private key (PEM-encoded). Must be set together with | **Default** | `""` (use system CA pool) | | **Env var** | `SQI_WORKER_NATS_TLS_CA_FILE` | -Path to the CA certificate used to verify the NATS server's TLS certificate -(PEM-encoded). Leave empty to use the system certificate pool. +Path to the CA certificate that verifies the **broker's** TLS certificate +(PEM-encoded), i.e. the certificate configured as `nats.tls.cert_file` on the +server. + +Empty means the system certificate pool; set means **that CA only**, so +pointing this at a farm CA pins the acceptable issuer rather than widening the +set. Distinct from +[`nats.server_tls_ca_file`](#natsserver_tls_ca_file), which verifies the HTTPS +API on a different port and may legitimately have a different issuer. ```yaml nats: @@ -265,13 +312,60 @@ whenever [`nats.join_token`](#natsjoin_token) or needs to enroll with no `server_url` set fails fast with an actionable error naming this field, rather than attempting a request with no host. +When the server has `http.tls` enabled this must be an `https://` URL, and +[`nats.server_tls_ca_file`](#natsserver_tls_ca_file) must name the CA that +signed its certificate. + +```yaml +nats: + server_url: "https://sqi-server.example:8080" +``` + +--- + +### `nats.server_tls_ca_file` + +| | | +|---|---| +| **Type** | `string` | +| **Default** | `""` (use system CA pool) | +| **Env var** | `SQI_WORKER_NATS_SERVER_TLS_CA_FILE` | + +Path to the CA certificate that verifies `sqi-server`'s HTTPS certificate at +[`nats.server_url`](#natsserver_url), for enrollment. + +Deliberately separate from [`nats.tls_ca_file`](#natstls_ca_file): the two +verify different certificates on different ports, which can legitimately come +from different issuers — a publicly-issued certificate on the API and a private +farm CA on the broker. On a farm generated by `sqi-server tls init` they are +simply the same `ca.crt`. + +**This is on the bootstrap critical path.** Enrollment runs over REST before the +worker holds any broker credential, so without it a farm with an HTTPS server +cannot bootstrap at all. Same CA semantics as everywhere: empty means the system +roots, set means that CA only. + ```yaml nats: - server_url: "http://sqi-server.example:8080" + server_tls_ca_file: "/etc/sqi/ca.crt" ``` --- +### `nats.server_tls_insecure_skip_verify` + +| | | +|---|---| +| **Type** | `bool` | +| **Default** | `false` | +| **Env var** | `SQI_WORKER_NATS_SERVER_TLS_INSECURE_SKIP_VERIFY` | + +Skip verification of `sqi-server`'s HTTPS certificate during enrollment. +**Development environments only** — it accepts any certificate, which makes the +enrollment request, join token included, interceptable. + +--- + ## `worker` — Identity and runtime behavior ### `worker.name` @@ -1433,6 +1527,12 @@ network by default. | **Default** | `"127.0.0.1:9091"` | | **Env var** | `SQI_WORKER_METRICS_ADDR` | +This listener is **plaintext unless [`metrics.tls.enabled`](#metricstlsenabled) +is set** — `nats.tls_*` covers the broker connection, not this one. It serves +metrics, health and (optionally) pprof, never credentials or job payloads, and +defaults to loopback. Turn TLS on together with any address reachable from +elsewhere. + TCP address the local HTTP server listens on. Use `0.0.0.0:9091` to expose the endpoints to Prometheus scrapers on the network (ensure the port is firewalled appropriately). @@ -1447,6 +1547,35 @@ metrics: --- +### `metrics.tls.enabled` + +| | | +|---|---| +| **Type** | `bool` | +| **Default** | `false` | +| **Env var** | `SQI_WORKER_METRICS_TLS_ENABLED` | + +Terminate TLS on the metrics/health listener. As with the server's listeners, +there is no plaintext port when it is on — the listener upgrades in place. + +Requires `metrics.tls.cert_file` and `metrics.tls.key_file` +(`SQI_WORKER_METRICS_TLS_CERT_FILE` / `SQI_WORKER_METRICS_TLS_KEY_FILE`); the +worker refuses to start without both, and refuses a pair it cannot load. The +worker needs a certificate of its own here — this listener is the worker's, not +the server's — so issue one with `sqi-server tls issue --host ` or +use whatever your monitoring stack already trusts. + +```yaml +metrics: + addr: "0.0.0.0:9091" + tls: + enabled: true + cert_file: "/etc/sqi/certs/worker.crt" + key_file: "/etc/sqi/certs/worker.key" +``` + +--- + ### `metrics.enable_pprof` | | | diff --git a/internal/api/auth_test.go b/internal/api/auth_test.go index 8e5c5310..a1ce1540 100644 --- a/internal/api/auth_test.go +++ b/internal/api/auth_test.go @@ -71,6 +71,17 @@ func seedAuthUser(t *testing.T, st store.Store, username, pw, role string) store // distinguish a same-origin browser call from a cross-site one riding the // ambient cookie. Set Origin from url's own scheme+host to model that. func doRequest(t *testing.T, method, url string, body any, cookie *http.Cookie) *http.Response { + t.Helper() + return doRequestWithClient(t, http.DefaultClient, method, url, body, cookie) +} + +// doRequestWithClient is [doRequest] with a caller-supplied client. A TLS +// httptest server needs its own client (http.DefaultClient cannot verify the +// self-signed certificate), and everything else about the request — the JSON +// body, the cookie, and the Origin header that goes with it — must stay +// identical, or a TLS test hits the CSRF middleware for reasons the test +// itself does not explain. +func doRequestWithClient(t *testing.T, client *http.Client, method, url string, body any, cookie *http.Cookie) *http.Response { t.Helper() var reader *bytes.Reader if body != nil { @@ -95,7 +106,7 @@ func doRequest(t *testing.T, method, url string, body any, cookie *http.Cookie) req.Header.Set("Origin", u.Scheme+"://"+u.Host) } } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { t.Fatalf("%s %s: %v", method, url, err) } diff --git a/internal/api/tlscookie_test.go b/internal/api/tlscookie_test.go new file mode 100644 index 00000000..d536ffd4 --- /dev/null +++ b/internal/api/tlscookie_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/uberware/sqi/internal/store/fake" +) + +// The session cookie's Secure attribute is resolved by authHandler.secure from +// the three-valued auth.session.cookie_secure setting. "auto" — the default — +// reads r.TLS. That logic predates TLS support and was, until these tests, +// pinned by nothing that ran against a real TLS listener: every existing +// cookie test sets CookieSecure to "false" explicitly. +// +// Both directions matter. A Secure cookie issued over plaintext is silently +// dropped by the browser and login simply stops working, with no error +// anywhere — so "not Secure on plaintext" needs a test just as much as +// "Secure under TLS" does. + +// loginCookieUnder starts srvFn's server against a router configured with the +// given cookie_secure mode, logs in, and returns the session cookie. +func loginCookieUnder(t *testing.T, tlsListener bool, mode string) *http.Cookie { + t.Helper() + st := fake.New() + seedAuthUser(t, st, "alice", "hunter2!", "operator") + + router := authRouterWith(st, func(d *Deps) { d.CookieSecure = mode }) + + var srv *httptest.Server + if tlsListener { + srv = httptest.NewTLSServer(router) + } else { + srv = httptest.NewServer(router) + } + t.Cleanup(srv.Close) + + client := srv.Client() // trusts httptest's own certificate when TLS + resp := doRequestWithClient(t, client, http.MethodPost, srv.URL+"/api/v1/auth/login", + map[string]string{"username": "alice", "password": "hunter2!"}, nil) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("login status = %d, want 200", resp.StatusCode) + } + c := sessionCookie(resp) + if c == nil { + t.Fatal("login did not set a sqi_session cookie") + } + return c +} + +func TestSessionCookie_SecureUnderTLS(t *testing.T) { + if c := loginCookieUnder(t, true, "auto"); !c.Secure { + t.Error("session cookie is not Secure on a TLS listener with cookie_secure=auto") + } +} + +func TestSessionCookie_NotSecureOnPlaintext(t *testing.T) { + if c := loginCookieUnder(t, false, "auto"); c.Secure { + t.Error("session cookie is Secure on a plaintext listener; browsers drop it and login breaks silently") + } +} + +func TestSessionCookie_ExplicitModesOverrideTransport(t *testing.T) { + if c := loginCookieUnder(t, false, "true"); !c.Secure { + t.Error("cookie_secure=true did not force Secure on a plaintext listener") + } + if c := loginCookieUnder(t, true, "false"); c.Secure { + t.Error("cookie_secure=false did not suppress Secure on a TLS listener") + } +} diff --git a/internal/bus/broker.go b/internal/bus/broker.go index 96290417..af37ec43 100644 --- a/internal/bus/broker.go +++ b/internal/bus/broker.go @@ -4,6 +4,7 @@ package bus import ( "context" + "crypto/tls" "errors" "fmt" "log/slog" @@ -17,6 +18,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/uberware/sqi/internal/brokerauth" + "github.com/uberware/sqi/internal/tlsutil" ) // BrokerConfig holds the parameters needed to start the embedded NATS server. @@ -45,6 +47,47 @@ type BrokerConfig struct { // Auth configures per-worker nkey authorization on the broker. Zero value // leaves authorization disabled. Auth BrokerAuthConfig + + // TLS configures transport encryption on the client listener. Zero value + // leaves the broker plaintext, which is the default. + TLS BrokerTLSConfig +} + +// buildBrokerTLS assembles the broker's server-side TLS configuration. +// The paths have already been validated at config load, so a failure here +// means the files changed underneath a running process. +// +// Note what is NOT here: the server's own connections need no client-side TLS +// configuration at all, because they never speak TLS. See adminOptions. +func buildBrokerTLS(cfg BrokerTLSConfig) (*tls.Config, error) { + tlsCfg, err := tlsutil.ServerConfig(cfg.CertFile, cfg.KeyFile, cfg.ClientCAFile) + if err != nil { + return nil, err + } + return tlsCfg, nil +} + +// BrokerTLSConfig controls TLS on the embedded broker's client listener. +// Zero value leaves TLS off, which is the default. +// +// This is bus's own view of the operator's nats.tls block, mapped by the +// caller, so that internal/bus does not import internal/config — the same +// boundary WorkerCredentialRef draws against internal/store. +type BrokerTLSConfig struct { + // Enabled makes the broker require TLS on every client connection. + Enabled bool + + // CertFile and KeyFile are the PEM certificate and key the broker + // presents. Both are validated at config load. + CertFile string + KeyFile string + + // ClientCAFile, when set, requires each connecting worker to present a + // client certificate signed by this CA. It is a transport control + // layered on the nkey credential, never a substitute: the certificate + // decides who may open a connection, the nkey decides which worker they + // are. + ClientCAFile string } // BrokerAuthConfig controls per-worker nkey authorization on the broker. @@ -160,6 +203,22 @@ func (b *Broker) Start(ctx context.Context) error { opts.Nkeys = buildNkeys(serverPub, b.cfg.Auth.Credentials, b.logger) } + if b.cfg.TLS.Enabled { + tlsCfg, err := buildBrokerTLS(b.cfg.TLS) + if err != nil { + return fmt.Errorf("bus: tls: %w", err) + } + // Setting TLSConfig is what makes the broker REQUIRE TLS: + // nats-server computes `tlsReq := opts.TLSConfig != nil`. There is no + // separate flag. TLSTimeout self-defaults to 2s when left zero. + // + // This must happen BEFORE the bootOpts clone below: ReloadCredentials + // rebuilds from that clone, so a TLSConfig set afterwards would be + // dropped the first time a worker is revoked — silently taking the + // whole broker back to plaintext. + opts.TLSConfig = tlsCfg + } + // Retain a pristine copy of the boot options for ReloadCredentials: // ReloadOptions documents that the Options passed to it must not be // reused, so credential reloads clone from this copy rather than the one @@ -198,7 +257,7 @@ func (b *Broker) Start(ctx context.Context) error { // Establish an admin connection used only for stream provisioning. // This is a plain TCP connection to the loopback listener; the latency // is negligible and avoids importing the server package into callers. - nc, err := nats.Connect(ns.ClientURL(), b.adminOptions()...) + nc, err := nats.Connect(inProcessURL, b.adminOptions(ns)...) if err != nil { ns.Shutdown() ns.WaitForShutdown() @@ -303,7 +362,7 @@ func (b *Broker) NewClient() (*Client, error) { if ns == nil { return nil, errors.New("bus: broker not started") } - return NewClient(ns.ClientURL(), b.logger, b.adminOptions()...) + return NewClient(inProcessURL, b.logger, b.adminOptions(ns)...) } // buildNkeys converts the enrolled credential set into NATS nkey users, plus @@ -366,18 +425,47 @@ func buildNkeys(serverPub string, creds []WorkerCredentialRef, logger *slog.Logg return users } -// adminOptions returns the connect options for the broker's own admin -// connection, which provisions streams. Empty when auth is disabled. -func (b *Broker) adminOptions() []nats.Option { - if !b.cfg.Auth.Enabled { - return nil - } +// adminOptions returns the connect options the server's OWN connections use — +// the admin connection that provisions streams, and every Client handed out by +// [Broker.NewClient]. +// +// These connections go in-process (nats.InProcessServer) rather than over the +// loopback TCP listener. That is what lets broker TLS be turned on without the +// server having to trust its own certificate to talk to itself: nats-server +// clears info.TLSRequired for in-process connections (server/server.go:3296). +// +// Authentication is unaffected — an in-process connection still traverses the +// normal client auth path, so the server nkey is presented and checked exactly +// as before. +func (b *Broker) adminOptions(ns *natsserver.Server) []nats.Option { + opts := []nats.Option{nats.InProcessServer(ns)} + b.mu.Lock() pub, seed := b.serverPub, b.serverSeed b.mu.Unlock() - return []nats.Option{brokerauth.NkeyOption(pub, seed)} + + if !b.cfg.Auth.Enabled { + return opts + } + return append(opts, brokerauth.NkeyOption(pub, seed)) } +// inProcessURL is the URL the server's own connections dial: none. +// +// It is deliberately empty even though nats.InProcessServer makes the URL +// unused for transport, because nats.go still PARSES it — and +// Server.ClientURL() returns a "tls://" scheme once TLS is enabled, which makes +// the client negotiate TLS over what is only a net.Pipe inside this process. +// That in turn would demand the server trust its own certificate, for a +// connection that never touches a socket. +// +// Passing no URL avoids all of it: nats-server clears info.TLSRequired for +// in-process connections (server/server.go:3296), so the connection is plain +// and authentication still applies — an in-process client presents the server +// nkey and is checked exactly like any other. Verified against a broker with +// TLS, mutual TLS and nkey auth all enabled. +const inProcessURL = "" + // ReloadCredentials replaces the enrolled worker set on a running broker. // // Revocation is synchronous, not eventually-consistent: nats-server's diff --git a/internal/bus/broker_inprocess_test.go b/internal/bus/broker_inprocess_test.go new file mode 100644 index 00000000..707a099f --- /dev/null +++ b/internal/bus/broker_inprocess_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package bus + +import ( + "context" + "crypto/tls" + "testing" + "time" + + nats "github.com/nats-io/nats.go" +) + +func TestBrokerInProcess_ServerReachesJetStreamUnderTLSAndAuth(t *testing.T) { + dir, _ := farmCerts(t) + enrolled, _ := enrolledWorker(t, "worker-01") + b := startBrokerTLS( + t, + brokerTLSOf(dir, false), + BrokerAuthConfig{Enabled: true, Credentials: []WorkerCredentialRef{enrolled}}, + ) + + // The server's own client carries no TLS configuration at all: the + // connection is an in-process pipe, which nats-server exempts from the TLS + // requirement, so there is nothing to trust and nothing to encrypt. + c, err := b.NewClient() + if err != nil { + t.Fatalf("NewClient under TLS + auth: %v", err) + } + defer c.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := b.Check(ctx); err != nil { + t.Errorf("broker health check failed with TLS and auth on: %v", err) + } +} + +func TestBrokerInProcess_NetworkClientsStillNeedCertificates(t *testing.T) { + dir, _ := farmCerts(t) + b := startBrokerTLS(t, brokerTLSOf(dir, true), BrokerAuthConfig{}) + + // The server's own connection is in-process and therefore plaintext, which + // bypasses the client-certificate requirement by never doing a handshake. + // A real network client must not get the same pass. + nc, err := nats.Connect(b.ClientURL(), nats.NoReconnect(), nats.Secure(&tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caPool(t, dir), + })) + if err == nil { + nc.Close() + t.Fatal("a TCP client without a certificate connected to an mTLS broker") + } +} + +func TestBrokerInProcess_PlaintextStillRefusedForNetworkClients(t *testing.T) { + dir, _ := farmCerts(t) + b := startBrokerTLS(t, brokerTLSOf(dir, false), BrokerAuthConfig{}) + + if nc, err := nats.Connect(b.ClientURL(), nats.NoReconnect()); err == nil { + nc.Close() + t.Fatal("plaintext TCP client connected to a TLS-required broker") + } +} diff --git a/internal/bus/broker_tls_test.go b/internal/bus/broker_tls_test.go new file mode 100644 index 00000000..91ab1da8 --- /dev/null +++ b/internal/bus/broker_tls_test.go @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package bus + +import ( + "context" + "crypto/tls" + "crypto/x509" + "log/slog" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + nats "github.com/nats-io/nats.go" + + "github.com/uberware/sqi/internal/certgen" +) + +// farmCerts generates a CA plus a broker server keypair covering loopback, +// and returns the directory holding ca.crt / server.crt / server.key along +// with the CA itself so a test can also mint client certificates. +func farmCerts(t *testing.T) (dir string, ca *certgen.CA) { + t.Helper() + dir = t.TempDir() + ca, err := certgen.NewCA("test farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"localhost", "127.0.0.1", "::1"}, 365*24*time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "server", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + return dir, ca +} + +// startBrokerTLS boots an embedded broker with the given TLS and auth +// configuration on an OS-assigned loopback port. +func startBrokerTLS(t *testing.T, tlsCfg BrokerTLSConfig, auth BrokerAuthConfig) *Broker { + t.Helper() + cfg := BrokerConfig{ + Addr: net.JoinHostPort("127.0.0.1", itoa(freePort(t))), + DataDir: t.TempDir() + "/nats", + MaxStoreMB: 64, + Auth: auth, + TLS: tlsCfg, + } + b := New(cfg, slog.New(slog.DiscardHandler)) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := b.Start(ctx); err != nil { + t.Fatalf("startBrokerTLS: Start: %v", err) + } + t.Cleanup(b.Shutdown) + return b +} + +// caPool returns a root pool trusting only the CA in dir. +func caPool(t *testing.T, dir string) *x509.CertPool { + t.Helper() + pem, err := os.ReadFile(filepath.Join(dir, "ca.crt")) + if err != nil { + t.Fatalf("read ca.crt: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("ca.crt did not append to pool") + } + return pool +} + +// brokerTLSOf builds the BrokerTLSConfig pointing at dir's server keypair. +func brokerTLSOf(dir string, clientCA bool) BrokerTLSConfig { + cfg := BrokerTLSConfig{ + Enabled: true, + CertFile: filepath.Join(dir, "server.crt"), + KeyFile: filepath.Join(dir, "server.key"), + } + if clientCA { + cfg.ClientCAFile = filepath.Join(dir, "ca.crt") + } + return cfg +} + +// clientCertOption loads a generated client keypair as a nats TLS option. +func clientCertOption(t *testing.T, ca *certgen.CA, pool *x509.CertPool, commonName string) nats.Option { + t.Helper() + leaf, err := ca.NewClientCert(commonName, 365*24*time.Hour) + if err != nil { + t.Fatalf("NewClientCert: %v", err) + } + pair, err := tls.X509KeyPair(leaf.CertPEM, leaf.KeyPEM) + if err != nil { + t.Fatalf("X509KeyPair: %v", err) + } + return nats.Secure(&tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: pool, + Certificates: []tls.Certificate{pair}, + }) +} + +func TestBrokerTLS_RequiredWhenEnabled(t *testing.T) { + dir, _ := farmCerts(t) + b := startBrokerTLS(t, brokerTLSOf(dir, false), BrokerAuthConfig{}) + + nc, err := nats.Connect(b.ClientURL(), nats.Secure(&tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caPool(t, dir), + })) + if err != nil { + t.Fatalf("TLS client could not connect: %v", err) + } + nc.Close() + + // Setting Options.TLSConfig is what makes nats-server require TLS; a + // plaintext client must be refused. + if plain, err := nats.Connect(b.ClientURL(), nats.NoReconnect()); err == nil { + plain.Close() + t.Fatal("plaintext client connected to a TLS-required broker") + } +} + +func TestBrokerTLS_WrongCARefused(t *testing.T) { + dir, _ := farmCerts(t) + otherDir, _ := farmCerts(t) + b := startBrokerTLS(t, brokerTLSOf(dir, false), BrokerAuthConfig{}) + + _, err := nats.Connect(b.ClientURL(), nats.NoReconnect(), nats.Secure(&tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caPool(t, otherDir), + })) + if err == nil { + t.Fatal("client trusting a different CA connected successfully") + } + if !strings.Contains(err.Error(), "certificate") { + t.Errorf("error = %v, want a certificate verification failure", err) + } +} + +func TestBrokerTLS_MTLSRequiresClientCert(t *testing.T) { + dir, ca := farmCerts(t) + b := startBrokerTLS(t, brokerTLSOf(dir, true), BrokerAuthConfig{}) + pool := caPool(t, dir) + + // Correct server trust but no client certificate: refused. + if nc, err := nats.Connect(b.ClientURL(), nats.NoReconnect(), nats.Secure(&tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: pool, + })); err == nil { + nc.Close() + t.Fatal("client without a certificate connected to an mTLS broker") + } + + // With a farm-issued client certificate: accepted. + nc, err := nats.Connect(b.ClientURL(), clientCertOption(t, ca, pool, "worker-01")) + if err != nil { + t.Fatalf("client with a valid certificate was refused: %v", err) + } + nc.Close() +} + +func TestBrokerTLS_MTLSLayersOnNkeyRatherThanReplacingIt(t *testing.T) { + dir, ca := farmCerts(t) + enrolled, _ := enrolledWorker(t, "worker-01") + b := startBrokerTLS( + t, + brokerTLSOf(dir, true), + BrokerAuthConfig{Enabled: true, Credentials: []WorkerCredentialRef{enrolled}}, + ) + pool := caPool(t, dir) + + // A valid farm client certificate, but no enrolled nkey. A certificate + // alone must not be sufficient: mTLS gates the transport, the nkey is + // still the identity. + nc, err := nats.Connect(b.ClientURL(), nats.NoReconnect(), clientCertOption(t, ca, pool, "impostor")) + if err == nil { + nc.Close() + t.Fatal("a farm client certificate alone authenticated; mTLS must layer on the nkey, not replace it") + } +} + +func TestBrokerTLS_OffIsUnchanged(t *testing.T) { + // The zero-value TLS field is the default path: plaintext, exactly as it + // behaved before TLS existed. + b := startBrokerTLS(t, BrokerTLSConfig{}, BrokerAuthConfig{}) + nc, err := nats.Connect(b.ClientURL()) + if err != nil { + t.Fatalf("plaintext client could not connect to a default broker: %v", err) + } + nc.Close() +} diff --git a/internal/certgen/certgen.go b/internal/certgen/certgen.go new file mode 100644 index 00000000..94e93687 --- /dev/null +++ b/internal/certgen/certgen.go @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package certgen generates the certificate material a farm needs to run +// sqi over TLS: a farm certificate authority, server certificates for the +// API listener and the embedded broker, and client certificates for the +// optional worker mTLS path. +// +// It exists because SANs and extended key usage are exactly what +// hand-rolled openssl invocations get wrong, and getting them wrong fails +// at worker-connect time — far from the mistake — rather than at load. +// +// The same generator backs `sqi-server tls init` and the test suites, so +// the code path under test is the code path that ships. Tests therefore +// never need checked-in certificate fixtures, which expire and break CI on +// a date nobody chose. +package certgen + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "time" +) + +// CA is a generated certificate authority held in memory. +type CA struct { + Cert *x509.Certificate + Key *ecdsa.PrivateKey + CertPEM []byte + KeyPEM []byte +} + +// Leaf is a generated end-entity certificate held in memory. +type Leaf struct { + CertPEM []byte + KeyPEM []byte +} + +// serialNumber returns a random 128-bit certificate serial number. +func serialNumber() (*big.Int, error) { + limit := new(big.Int).Lsh(big.NewInt(1), 128) + n, err := rand.Int(rand.Reader, limit) + if err != nil { + return nil, fmt.Errorf("certgen: generate serial: %w", err) + } + return n, nil +} + +// encode marshals a certificate DER blob and its key into PEM form. +func encode(der []byte, key *ecdsa.PrivateKey) (certPEM, keyPEM []byte, err error) { + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return nil, nil, fmt.Errorf("certgen: marshal key: %w", err) + } + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM, nil +} + +// NewCA generates a self-signed certificate authority valid for validFor. +func NewCA(commonName string, validFor time.Duration) (*CA, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("certgen: generate CA key: %w", err) + } + serial, err := serialNumber() + if err != nil { + return nil, err + } + now := time.Now() + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: commonName}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(validFor), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLenZero: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, fmt.Errorf("certgen: create CA certificate: %w", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("certgen: parse CA certificate: %w", err) + } + certPEM, keyPEM, err := encode(der, key) + if err != nil { + return nil, err + } + return &CA{Cert: cert, Key: key, CertPEM: certPEM, KeyPEM: keyPEM}, nil +} + +// LoadCA reconstructs a [CA] from PEM material previously written by +// [WriteCA], so certificates can be issued from a farm CA that already exists. +// +// Without this, `tls init` is the only way to get a CA and it refuses to +// overwrite one — which left no way at all to add a worker to an established +// mTLS farm, or to rotate a server certificate, short of replacing the CA and +// reissuing everything. +func LoadCA(certPEM, keyPEM []byte) (*CA, error) { + certBlock, _ := pem.Decode(certPEM) + if certBlock == nil || certBlock.Type != "CERTIFICATE" { + return nil, errors.New("certgen: no CERTIFICATE block in the CA certificate") + } + cert, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("certgen: parse CA certificate: %w", err) + } + // A leaf cannot sign. Accepting one would mint certificates that every peer + // rejects, discovered only at the first handshake. + if !cert.IsCA { + return nil, errors.New("certgen: the certificate is not a CA (BasicConstraints CA:FALSE)") + } + + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, errors.New("certgen: no PEM block in the CA key") + } + key, err := parseECKey(keyBlock) + if err != nil { + return nil, err + } + + // The pair must actually belong together: a mismatched key produces + // signatures nothing can verify. + pub, ok := cert.PublicKey.(*ecdsa.PublicKey) + if !ok { + return nil, errors.New("certgen: CA certificate does not carry an ECDSA public key") + } + if !pub.Equal(key.Public()) { + return nil, errors.New("certgen: CA key does not match the CA certificate") + } + + return &CA{Cert: cert, Key: key, CertPEM: certPEM, KeyPEM: keyPEM}, nil +} + +// parseECKey accepts the EC key encodings openssl and Go produce. +func parseECKey(block *pem.Block) (*ecdsa.PrivateKey, error) { + if key, err := x509.ParseECPrivateKey(block.Bytes); err == nil { + return key, nil + } + generic, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("certgen: parse CA key: %w", err) + } + key, ok := generic.(*ecdsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("certgen: CA key is %T, want an ECDSA key", generic) + } + return key, nil +} + +// newLeaf signs an end-entity certificate with ca. +func (ca *CA) newLeaf(commonName string, hosts []string, usage x509.ExtKeyUsage, validFor time.Duration) (*Leaf, error) { + return ca.newLeafAt(commonName, hosts, usage, time.Now().Add(-time.Hour), validFor+time.Hour) +} + +// newLeafAt is newLeaf with an explicit validity start. +func (ca *CA) newLeafAt(commonName string, hosts []string, usage x509.ExtKeyUsage, notBefore time.Time, validFor time.Duration) (*Leaf, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("certgen: generate leaf key: %w", err) + } + serial, err := serialNumber() + if err != nil { + return nil, err + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: commonName}, + NotBefore: notBefore, + NotAfter: notBefore.Add(validFor), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{usage}, + BasicConstraintsValid: true, + } + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + continue + } + tmpl.DNSNames = append(tmpl.DNSNames, h) + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, &key.PublicKey, ca.Key) + if err != nil { + return nil, fmt.Errorf("certgen: create leaf certificate: %w", err) + } + certPEM, keyPEM, err := encode(der, key) + if err != nil { + return nil, err + } + return &Leaf{CertPEM: certPEM, KeyPEM: keyPEM}, nil +} + +// NewServerCert signs a server certificate covering hosts. Entries that +// parse as an IP address become IP SANs; the rest become DNS SANs. +// +// The validity window starts an hour in the past, so a certificate is usable +// immediately on a host whose clock runs slightly behind the issuer's. +func (ca *CA) NewServerCert(hosts []string, validFor time.Duration) (*Leaf, error) { + return ca.NewServerCertNotBefore(hosts, time.Now().Add(-time.Hour), validFor+time.Hour) +} + +// NewServerCertNotBefore is [CA.NewServerCert] with an explicit start to the +// validity window, for a certificate minted ahead of a scheduled rollout. +func (ca *CA) NewServerCertNotBefore(hosts []string, notBefore time.Time, validFor time.Duration) (*Leaf, error) { + cn := "sqi-server" + if len(hosts) > 0 { + cn = hosts[0] + } + return ca.newLeafAt(cn, hosts, x509.ExtKeyUsageServerAuth, notBefore, validFor) +} + +// NewClientCert signs a client certificate for the worker mTLS path. The +// common name is informational: sqi does not bind a certificate to a worker +// identity — the nkey does that. See docs/tls.md. +func (ca *CA) NewClientCert(commonName string, validFor time.Duration) (*Leaf, error) { + return ca.newLeaf(commonName, nil, x509.ExtKeyUsageClientAuth, validFor) +} diff --git a/internal/certgen/certgen_test.go b/internal/certgen/certgen_test.go new file mode 100644 index 00000000..5b145117 --- /dev/null +++ b/internal/certgen/certgen_test.go @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package certgen_test + +import ( + "crypto/x509" + "encoding/pem" + "net" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" +) + +func parseLeaf(t *testing.T, pemBytes []byte) *x509.Certificate { + t.Helper() + block, _ := pem.Decode(pemBytes) + if block == nil { + t.Fatal("no PEM block in certificate") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parse certificate: %v", err) + } + return cert +} + +func TestNewCA_IsACertificateAuthority(t *testing.T) { + ca, err := certgen.NewCA("sqi farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + cert := parseLeaf(t, ca.CertPEM) + if !cert.IsCA { + t.Error("CA certificate has IsCA = false, want true") + } + if cert.KeyUsage&x509.KeyUsageCertSign == 0 { + t.Error("CA certificate lacks KeyUsageCertSign") + } +} + +func TestNewServerCert_SANsAndExtKeyUsage(t *testing.T) { + ca, err := certgen.NewCA("sqi farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"sqi.example", "127.0.0.1", "::1"}, 2*365*24*time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + cert := parseLeaf(t, leaf.CertPEM) + + if got := cert.DNSNames; len(got) != 1 || got[0] != "sqi.example" { + t.Errorf("DNSNames = %v, want [sqi.example]", got) + } + if len(cert.IPAddresses) != 2 { + t.Fatalf("IPAddresses = %v, want 2 entries", cert.IPAddresses) + } + if !cert.IPAddresses[0].Equal(net.ParseIP("127.0.0.1")) { + t.Errorf("IPAddresses[0] = %v, want 127.0.0.1", cert.IPAddresses[0]) + } + if len(cert.ExtKeyUsage) != 1 || cert.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth { + t.Errorf("ExtKeyUsage = %v, want [ServerAuth]", cert.ExtKeyUsage) + } +} + +func TestNewClientCert_ExtKeyUsageClientAuth(t *testing.T) { + ca, err := certgen.NewCA("sqi farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + leaf, err := ca.NewClientCert("worker-01", 2*365*24*time.Hour) + if err != nil { + t.Fatalf("NewClientCert: %v", err) + } + cert := parseLeaf(t, leaf.CertPEM) + if len(cert.ExtKeyUsage) != 1 || cert.ExtKeyUsage[0] != x509.ExtKeyUsageClientAuth { + t.Errorf("ExtKeyUsage = %v, want [ClientAuth]", cert.ExtKeyUsage) + } + if cert.Subject.CommonName != "worker-01" { + t.Errorf("CommonName = %q, want %q", cert.Subject.CommonName, "worker-01") + } +} + +func TestNewServerCert_ChainVerifiesAgainstCA(t *testing.T) { + ca, err := certgen.NewCA("sqi farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"sqi.example"}, 2*365*24*time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca.CertPEM) { + t.Fatal("CA PEM did not append to pool") + } + cert := parseLeaf(t, leaf.CertPEM) + if _, err := cert.Verify(x509.VerifyOptions{ + Roots: pool, + DNSName: "sqi.example", + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("leaf does not verify against its CA: %v", err) + } +} + +func TestNewServerCert_ExpiredWhenValidForIsNegative(t *testing.T) { + ca, err := certgen.NewCA("sqi farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"sqi.example"}, -1*time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + cert := parseLeaf(t, leaf.CertPEM) + if !cert.NotAfter.Before(time.Now()) { + t.Errorf("NotAfter = %v, want a time in the past", cert.NotAfter) + } +} + +func TestLoadCA_RoundTripsAndIssues(t *testing.T) { + original, err := certgen.NewCA("sqi farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + + // Reload from PEM exactly as an operator's ca.crt/ca.key would be. + loaded, err := certgen.LoadCA(original.CertPEM, original.KeyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + if loaded.Cert.Subject.CommonName != "sqi farm CA" { + t.Errorf("CommonName = %q, want %q", loaded.Cert.Subject.CommonName, "sqi farm CA") + } + + // The whole point: a certificate issued from the RELOADED CA must verify + // against the ORIGINAL one. Anything less means an operator adding a worker + // later gets a certificate their farm rejects. + leaf, err := loaded.NewClientCert("render-07", 2*365*24*time.Hour) + if err != nil { + t.Fatalf("NewClientCert from a loaded CA: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(original.CertPEM) { + t.Fatal("original CA PEM did not append to pool") + } + cert := parseLeaf(t, leaf.CertPEM) + if _, err := cert.Verify(x509.VerifyOptions{ + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }); err != nil { + t.Errorf("certificate issued from the reloaded CA does not verify against the original: %v", err) + } +} + +func TestLoadCA_RejectsBadInput(t *testing.T) { + good, err := certgen.NewCA("sqi farm CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + other, err := certgen.NewCA("other CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + + tests := []struct { + name string + certPEM, keyPEM []byte + }{ + {"garbage certificate", []byte("not a pem"), good.KeyPEM}, + {"garbage key", good.CertPEM, []byte("not a pem")}, + {"key from a different CA", good.CertPEM, other.KeyPEM}, + {"empty", nil, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := certgen.LoadCA(tt.certPEM, tt.keyPEM); err == nil { + t.Error("LoadCA accepted invalid material") + } + }) + } +} + +func TestLoadCA_RejectsANonCACertificate(t *testing.T) { + ca, err := certgen.NewCA("sqi farm CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"localhost"}, time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + // A leaf cannot sign. Loading one as a CA would produce certificates that + // nothing accepts, discovered only at the first handshake. + if _, err := certgen.LoadCA(leaf.CertPEM, leaf.KeyPEM); err == nil { + t.Error("LoadCA accepted a non-CA certificate") + } +} diff --git a/internal/certgen/write.go b/internal/certgen/write.go new file mode 100644 index 00000000..672fb9e4 --- /dev/null +++ b/internal/certgen/write.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package certgen + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// ErrCAExists is returned by WriteCA when dir already holds a CA key. +// Regenerating a farm CA in place invalidates every certificate issued from +// it, so the caller must move the old one aside deliberately. +var ErrCAExists = errors.New("certgen: CA already exists") + +const ( + certMode os.FileMode = 0o644 + keyMode os.FileMode = 0o600 +) + +// writePair writes a certificate and its private key with the right modes. +func writePair(dir, base string, certPEM, keyPEM []byte) error { + // 0750: the directory holds private keys, so it must not be world-readable. + if err := os.MkdirAll(dir, 0o750); err != nil { + return fmt.Errorf("certgen: create %s: %w", dir, err) + } + certPath := filepath.Join(dir, base+".crt") + keyPath := filepath.Join(dir, base+".key") + if err := os.WriteFile(certPath, certPEM, certMode); err != nil { + return fmt.Errorf("certgen: write %s: %w", certPath, err) + } + if err := os.WriteFile(keyPath, keyPEM, keyMode); err != nil { + return fmt.Errorf("certgen: write %s: %w", keyPath, err) + } + return nil +} + +// WriteCA writes ca.crt (0644) and ca.key (0600) into dir. It refuses to +// overwrite an existing ca.key. +func WriteCA(dir string, ca *CA) error { + keyPath := filepath.Join(dir, "ca.key") + if _, err := os.Stat(keyPath); err == nil { + return fmt.Errorf("%w at %s; to add a worker or rotate the server certificate use `sqi-server tls issue`, "+ + "which signs from this CA — replacing the CA itself means moving it aside first, and every "+ + "certificate ever issued from it stops verifying", ErrCAExists, keyPath) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("certgen: stat %s: %w", keyPath, err) + } + return writePair(dir, "ca", ca.CertPEM, ca.KeyPEM) +} + +// WriteLeaf writes .crt (0644) and .key (0600) into dir. +func WriteLeaf(dir, name string, leaf *Leaf) error { + return writePair(dir, name, leaf.CertPEM, leaf.KeyPEM) +} diff --git a/internal/certgen/write_test.go b/internal/certgen/write_test.go new file mode 100644 index 00000000..42f88fc3 --- /dev/null +++ b/internal/certgen/write_test.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package certgen_test + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" +) + +func TestWriteCA_FileModes(t *testing.T) { + dir := t.TempDir() + ca, err := certgen.NewCA("sqi farm CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + + for name, want := range map[string]os.FileMode{ + "ca.crt": 0o644, + "ca.key": 0o600, + } { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("stat %s: %v", name, err) + } + if got := info.Mode().Perm(); got != want { + t.Errorf("%s mode = %04o, want %04o", name, got, want) + } + } +} + +func TestWriteCA_RefusesToOverwriteExistingCA(t *testing.T) { + dir := t.TempDir() + first, err := certgen.NewCA("sqi farm CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, first); err != nil { + t.Fatalf("WriteCA (first): %v", err) + } + original, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + t.Fatalf("read ca.key: %v", err) + } + + second, err := certgen.NewCA("sqi farm CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + err = certgen.WriteCA(dir, second) + if !errors.Is(err, certgen.ErrCAExists) { + t.Fatalf("WriteCA (second) error = %v, want ErrCAExists", err) + } + + after, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + t.Fatalf("re-read ca.key: %v", err) + } + if string(after) != string(original) { + t.Error("ca.key was modified despite the refusal; a replaced farm CA invalidates every certificate issued from it") + } +} + +func TestWriteLeaf_FileModes(t *testing.T) { + dir := t.TempDir() + ca, err := certgen.NewCA("sqi farm CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"sqi.example"}, time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "server", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + + for name, want := range map[string]os.FileMode{ + "server.crt": 0o644, + "server.key": 0o600, + } { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("stat %s: %v", name, err) + } + if got := info.Mode().Perm(); got != want { + t.Errorf("%s mode = %04o, want %04o", name, got, want) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 68b4a3b9..acbee1e8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -62,6 +62,9 @@ type HTTPConfig struct { // separately-hosted UI must name its origin explicitly here. // Env: SQI_HTTP_CORS_ORIGINS (comma-separated) CORSOrigins []string `yaml:"cors_origins"` + + // TLS terminates HTTPS in-process on this listener. Off by default. + TLS TLSConfig `yaml:"tls"` } // NATSConfig controls the embedded NATS JetStream broker. @@ -86,6 +89,53 @@ type NATSConfig struct { // different surfaces, and coupling them would force an operator who // wants worker authentication into user accounts they did not ask for. Auth NATSAuthConfig `yaml:"auth"` + + // TLS encrypts the embedded broker's client listener. Off by default. + TLS NATSTLSConfig `yaml:"tls"` +} + +// TLSConfig controls in-process TLS termination for a listener. Zero value +// leaves TLS off, which is the default and must stay byte-for-byte +// equivalent to how the listener behaved before TLS existed. +type TLSConfig struct { + // Enabled turns the listener into an HTTPS listener. There is no + // plaintext port when it is on: the listener upgrades in place. + // Env: SQI_HTTP_TLS_ENABLED + Enabled bool `yaml:"enabled"` + + // CertFile is the PEM certificate (leaf first, then any intermediates). + // Env: SQI_HTTP_TLS_CERT_FILE + CertFile string `yaml:"cert_file"` + + // KeyFile is the PEM private key matching CertFile. + // Env: SQI_HTTP_TLS_KEY_FILE + KeyFile string `yaml:"key_file"` +} + +// NATSTLSConfig controls TLS on the embedded broker. It is deliberately +// separate from the http.tls block: the two protect different surfaces and +// an operator may legitimately use a publicly-issued certificate on the API +// and a private farm CA on the broker. +type NATSTLSConfig struct { + // Enabled requires TLS on every broker connection. + // Env: SQI_NATS_TLS_ENABLED + Enabled bool `yaml:"enabled"` + + // CertFile is the PEM certificate presented to connecting workers. + // Env: SQI_NATS_TLS_CERT_FILE + CertFile string `yaml:"cert_file"` + + // KeyFile is the PEM private key matching CertFile. + // Env: SQI_NATS_TLS_KEY_FILE + KeyFile string `yaml:"key_file"` + + // ClientCAFile, when set, requires every worker to present a client + // certificate signed by this CA. This is a TRANSPORT control layered on + // top of the per-worker nkey credential, not a replacement for it: the + // certificate gates who may open a connection, the nkey decides which + // worker you are. + // Env: SQI_NATS_TLS_CLIENT_CA_FILE + ClientCAFile string `yaml:"client_ca_file"` } // NATSAuthConfig configures broker authentication and worker enrollment. diff --git a/internal/config/loader.go b/internal/config/loader.go index a2867522..56e50fe4 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -152,6 +152,11 @@ type fileConfig struct { Addr *string `yaml:"addr"` EnablePprof *bool `yaml:"enable_pprof"` CORSOrigins *[]string `yaml:"cors_origins"` + TLS *struct { + Enabled *bool `yaml:"enabled"` + CertFile *string `yaml:"cert_file"` + KeyFile *string `yaml:"key_file"` + } `yaml:"tls"` } `yaml:"http"` NATS *struct { @@ -164,6 +169,12 @@ type fileConfig struct { JoinTokenSingleUse *bool `yaml:"join_token_single_use"` EnrollmentEndpointEnabled *bool `yaml:"enrollment_endpoint_enabled"` } `yaml:"auth"` + TLS *struct { + Enabled *bool `yaml:"enabled"` + CertFile *string `yaml:"cert_file"` + KeyFile *string `yaml:"key_file"` + ClientCAFile *string `yaml:"client_ca_file"` + } `yaml:"tls"` } `yaml:"nats"` Store *struct { @@ -335,6 +346,24 @@ func mergeHTTPFile(cfg *Config, fc fileConfig) { if fc.HTTP.CORSOrigins != nil { cfg.HTTP.CORSOrigins = *fc.HTTP.CORSOrigins } + mergeHTTPTLSFile(cfg, fc.HTTP.TLS) +} + +// mergeHTTPTLSFile overlays the http.tls sub-fields from fc onto cfg. Split +// out of [mergeHTTPFile] to keep its cyclomatic complexity under the lint +// threshold, following the mergeNATSAuthFile precedent. +func mergeHTTPTLSFile(cfg *Config, t *struct { + Enabled *bool `yaml:"enabled"` + CertFile *string `yaml:"cert_file"` + KeyFile *string `yaml:"key_file"` +}, +) { + if t == nil { + return + } + setIfNotNilBool(&cfg.HTTP.TLS.Enabled, t.Enabled) + setIfNotNilString(&cfg.HTTP.TLS.CertFile, t.CertFile) + setIfNotNilString(&cfg.HTTP.TLS.KeyFile, t.KeyFile) } func mergeNATSFile(cfg *Config, fc fileConfig) { @@ -351,6 +380,25 @@ func mergeNATSFile(cfg *Config, fc fileConfig) { cfg.NATS.MaxStoreMB = *fc.NATS.MaxStoreMB } mergeNATSAuthFile(cfg, fc.NATS.Auth) + mergeNATSTLSFile(cfg, fc.NATS.TLS) +} + +// mergeNATSTLSFile overlays the nats.tls sub-fields from fc onto cfg. Split +// out of [mergeNATSFile] for the same reason as [mergeNATSAuthFile]. +func mergeNATSTLSFile(cfg *Config, t *struct { + Enabled *bool `yaml:"enabled"` + CertFile *string `yaml:"cert_file"` + KeyFile *string `yaml:"key_file"` + ClientCAFile *string `yaml:"client_ca_file"` +}, +) { + if t == nil { + return + } + setIfNotNilBool(&cfg.NATS.TLS.Enabled, t.Enabled) + setIfNotNilString(&cfg.NATS.TLS.CertFile, t.CertFile) + setIfNotNilString(&cfg.NATS.TLS.KeyFile, t.KeyFile) + setIfNotNilString(&cfg.NATS.TLS.ClientCAFile, t.ClientCAFile) } // mergeNATSAuthFile overlays the nats.auth sub-fields from fc onto cfg. Split @@ -710,11 +758,18 @@ func applyEnv(cfg *Config) error { setString(&cfg.HTTP.Addr, "SQI_HTTP_ADDR") collect(setBool(&cfg.HTTP.EnablePprof, "SQI_HTTP_ENABLE_PPROF")) + collect(setBool(&cfg.HTTP.TLS.Enabled, "SQI_HTTP_TLS_ENABLED")) + setString(&cfg.HTTP.TLS.CertFile, "SQI_HTTP_TLS_CERT_FILE") + setString(&cfg.HTTP.TLS.KeyFile, "SQI_HTTP_TLS_KEY_FILE") setStringSlice(&cfg.HTTP.CORSOrigins, "SQI_HTTP_CORS_ORIGINS") setString(&cfg.NATS.Addr, "SQI_NATS_ADDR") setString(&cfg.NATS.DataDir, "SQI_NATS_DATA_DIR") collect(setInt(&cfg.NATS.MaxStoreMB, "SQI_NATS_MAX_STORE_MB")) + collect(setBool(&cfg.NATS.TLS.Enabled, "SQI_NATS_TLS_ENABLED")) + setString(&cfg.NATS.TLS.CertFile, "SQI_NATS_TLS_CERT_FILE") + setString(&cfg.NATS.TLS.KeyFile, "SQI_NATS_TLS_KEY_FILE") + setString(&cfg.NATS.TLS.ClientCAFile, "SQI_NATS_TLS_CLIENT_CA_FILE") collect(setBool(&cfg.NATS.Auth.Enabled, "SQI_NATS_AUTH_ENABLED")) collect(setDuration(&cfg.NATS.Auth.JoinTokenTTL, "SQI_NATS_AUTH_JOIN_TOKEN_TTL")) collect(setBool(&cfg.NATS.Auth.JoinTokenSingleUse, "SQI_NATS_AUTH_JOIN_TOKEN_SINGLE_USE")) diff --git a/internal/config/tls.go b/internal/config/tls.go new file mode 100644 index 00000000..b8f467f6 --- /dev/null +++ b/internal/config/tls.go @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "time" + + "github.com/uberware/sqi/internal/tlsutil" +) + +// expiryWarnWindow is how close to NotAfter a certificate has to be before +// startup warns about it. +const expiryWarnWindow = 30 * 24 * time.Hour + +// missingPathErrors reports the "must be set" errors for an enabled TLS block +// with an empty certificate or key path. +func missingPathErrors(certField, keyField, certFile, keyFile string) []ValidationError { + var errs []ValidationError + msg := fmt.Sprintf("must be set when TLS is enabled; set %s and %s, or generate a pair with `sqi-server tls init`", certField, keyField) + if certFile == "" { + errs = append(errs, ValidationError{Field: certField, Message: msg}) + } + if keyFile == "" { + errs = append(errs, ValidationError{Field: keyField, Message: msg}) + } + return errs +} + +// validateKeypair checks that certFile and keyFile load together and that the +// leaf's validity window covers now. +// +// This covers loadability, pairing and validity dates — NOT every problem a +// certificate can have. A SAN or hostname mismatch is still only discovered +// when a peer verifies the certificate, because it depends on the name that +// peer used, which is not knowable here. +func validateKeypair(certField, keyField, certFile, keyFile string) []ValidationError { + if certFile == "" || keyFile == "" { + return missingPathErrors(certField, keyField, certFile, keyFile) + } + + pair, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + msg := err.Error() + if os.IsNotExist(err) { + msg = fmt.Sprintf("no such file: %s", err) + } + return []ValidationError{{ + Field: certField, + Message: fmt.Sprintf("cannot load certificate/key pair (%s, %s): %s", certFile, keyFile, msg), + }} + } + + leaf, err := tlsutil.LeafOf(pair) + if err != nil { + return []ValidationError{{ + Field: certField, + Message: fmt.Sprintf("cannot parse leaf certificate in %s: %s", certFile, err), + }} + } + now := time.Now() + if now.After(leaf.NotAfter) { + return []ValidationError{{ + Field: certField, + Message: fmt.Sprintf("certificate %s expired at %s; issue a new one (`sqi-server tls init`) — a server that boots with an expired certificate is a server whose whole farm fails to connect", certFile, leaf.NotAfter.Format(time.RFC3339)), + }} + } + // Not yet valid fails exactly like expired at the peer, and for a reason + // (a clock skew, a certificate minted for a future rollout) that is even + // harder to read from a handshake error. + if now.Before(leaf.NotBefore) { + return []ValidationError{{ + Field: certField, + Message: fmt.Sprintf("certificate %s is not valid until %s; check the system clock, or wait", certFile, leaf.NotBefore.Format(time.RFC3339)), + }} + } + return nil +} + +// validateHTTPTLS checks the http.tls block. +func validateHTTPTLS(cfg TLSConfig) []ValidationError { + if !cfg.Enabled { + return nil + } + return validateKeypair("http.tls.cert_file", "http.tls.key_file", cfg.CertFile, cfg.KeyFile) +} + +// validateNATSTLS checks the nats.tls block. +func validateNATSTLS(cfg NATSTLSConfig) []ValidationError { + if !cfg.Enabled { + if cfg.ClientCAFile != "" { + return []ValidationError{{ + Field: "nats.tls.client_ca_file", + Message: "requires nats.tls.enabled: true; client certificates cannot be verified on a plaintext listener", + }} + } + return nil + } + errs := validateKeypair("nats.tls.cert_file", "nats.tls.key_file", cfg.CertFile, cfg.KeyFile) + if cfg.ClientCAFile != "" { + if _, err := tlsutil.CertPool(cfg.ClientCAFile); err != nil { + errs = append(errs, ValidationError{ + Field: "nats.tls.client_ca_file", + Message: err.Error(), + }) + } + } + return errs +} + +// ExpiringSoon reports whether the leaf certificate in certFile expires +// within 30 days, returning its NotAfter. +// +// It inspects the FIRST CERTIFICATE block, which the docs require to be the +// leaf. A chain written the other way round would have this report the CA's +// expiry instead — harmless, but it would warn about the wrong date. It is advisory: startup warns, it +// does not refuse. Errors are swallowed because Validate has already rejected +// anything unloadable. +// +// It takes only the certificate: the private key has no bearing on expiry, and +// asking for it would imply otherwise. +func ExpiringSoon(certFile string) (time.Time, bool) { + raw, err := os.ReadFile(certFile) + if err != nil { + return time.Time{}, false + } + block, _ := pemDecode(raw) + if block == nil { + return time.Time{}, false + } + leaf, err := x509.ParseCertificate(block) + if err != nil { + return time.Time{}, false + } + return leaf.NotAfter, time.Until(leaf.NotAfter) < expiryWarnWindow +} + +// pemDecode returns the DER bytes of the first CERTIFICATE block in raw. +func pemDecode(raw []byte) ([]byte, bool) { + for len(raw) > 0 { + block, rest := pem.Decode(raw) + if block == nil { + return nil, false + } + if block.Type == "CERTIFICATE" { + return block.Bytes, true + } + raw = rest + } + return nil, false +} diff --git a/internal/config/tls_test.go b/internal/config/tls_test.go new file mode 100644 index 00000000..6793dbe8 --- /dev/null +++ b/internal/config/tls_test.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" + "github.com/uberware/sqi/internal/config" +) + +// writeCerts generates a farm CA and a server keypair into a temp dir and +// returns (certFile, keyFile, caFile). validFor is passed through so a +// caller can mint an already-expired certificate. +func writeCerts(t *testing.T, validFor time.Duration) (certFile, keyFile, caFile string) { + t.Helper() + dir := t.TempDir() + ca, err := certgen.NewCA("test CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"localhost", "127.0.0.1"}, validFor) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "server", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + return filepath.Join(dir, "server.crt"), filepath.Join(dir, "server.key"), filepath.Join(dir, "ca.crt") +} + +// writeFutureCerts mints a certificate whose validity window has not started, +// for the clock-skew case. certgen backdates NotBefore by an hour, so the +// window has to be pushed further out than that. +func writeFutureCerts(t *testing.T) (certFile, keyFile string) { + t.Helper() + dir := t.TempDir() + ca, err := certgen.NewCA("test CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + leaf, err := ca.NewServerCertNotBefore([]string{"localhost"}, time.Now().Add(48*time.Hour), 72*time.Hour) + if err != nil { + t.Fatalf("NewServerCertNotBefore: %v", err) + } + if err := certgen.WriteLeaf(dir, "future", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + return filepath.Join(dir, "future.crt"), filepath.Join(dir, "future.key") +} + +// findErr returns the ValidationError for field, or nil. +func findErr(errs []config.ValidationError, field string) *config.ValidationError { + for i := range errs { + if errs[i].Field == field { + return &errs[i] + } + } + return nil +} + +func TestValidate_HTTPTLS(t *testing.T) { + goodCert, goodKey, _ := writeCerts(t, time.Hour) + expiredCert, expiredKey, _ := writeCerts(t, -time.Hour) + futureCert, futureKey := writeFutureCerts(t) + otherCert, _, _ := writeCerts(t, time.Hour) + + tests := []struct { + name string + tlsCfg config.TLSConfig + wantField string + wantMatch string + }{ + { + name: "enabled with no cert or key", + tlsCfg: config.TLSConfig{Enabled: true}, + wantField: "http.tls.cert_file", + wantMatch: "http.tls.key_file", + }, + { + name: "enabled with cert but no key", + tlsCfg: config.TLSConfig{Enabled: true, CertFile: goodCert}, + wantField: "http.tls.key_file", + wantMatch: "must be set", + }, + { + name: "missing file", + tlsCfg: config.TLSConfig{Enabled: true, CertFile: goodCert + ".nope", KeyFile: goodKey}, + wantField: "http.tls.cert_file", + wantMatch: "no such file", + }, + { + name: "key does not match certificate", + tlsCfg: config.TLSConfig{Enabled: true, CertFile: otherCert, KeyFile: goodKey}, + wantField: "http.tls.cert_file", + wantMatch: "does not match", + }, + { + name: "not yet valid certificate", + tlsCfg: config.TLSConfig{Enabled: true, CertFile: futureCert, KeyFile: futureKey}, + wantField: "http.tls.cert_file", + wantMatch: "not valid until", + }, + { + name: "expired certificate", + tlsCfg: config.TLSConfig{Enabled: true, CertFile: expiredCert, KeyFile: expiredKey}, + wantField: "http.tls.cert_file", + wantMatch: "expired", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := config.DefaultConfig() + cfg.HTTP.TLS = tt.tlsCfg + errs := config.Validate(cfg) + ve := findErr(errs, tt.wantField) + if ve == nil { + t.Fatalf("no ValidationError for %q; got %v", tt.wantField, errs) + } + if !strings.Contains(ve.Message, tt.wantMatch) { + t.Errorf("message = %q, want it to contain %q", ve.Message, tt.wantMatch) + } + }) + } +} + +func TestValidate_HTTPTLSValidConfigPasses(t *testing.T) { + cert, key, _ := writeCerts(t, time.Hour) + cfg := config.DefaultConfig() + cfg.HTTP.TLS = config.TLSConfig{Enabled: true, CertFile: cert, KeyFile: key} + if errs := config.Validate(cfg); len(errs) != 0 { + t.Errorf("Validate() = %v, want no errors", errs) + } +} + +func TestValidate_NATSTLSClientCARequiresTLSEnabled(t *testing.T) { + _, _, caFile := writeCerts(t, time.Hour) + cfg := config.DefaultConfig() + cfg.NATS.TLS = config.NATSTLSConfig{Enabled: false, ClientCAFile: caFile} + + ve := findErr(config.Validate(cfg), "nats.tls.client_ca_file") + if ve == nil { + t.Fatal("client_ca_file without nats.tls.enabled was accepted; mTLS without TLS is meaningless") + } + if !strings.Contains(ve.Message, "nats.tls.enabled") { + t.Errorf("message = %q, want it to name nats.tls.enabled", ve.Message) + } +} + +func TestValidate_NATSTLSClientCAMustParse(t *testing.T) { + cert, key, _ := writeCerts(t, time.Hour) + notACA := filepath.Join(t.TempDir(), "garbage.pem") + if err := os.WriteFile(notACA, []byte("this is not a certificate\n"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + cfg := config.DefaultConfig() + cfg.NATS.TLS = config.NATSTLSConfig{ + Enabled: true, CertFile: cert, KeyFile: key, ClientCAFile: notACA, + } + if ve := findErr(config.Validate(cfg), "nats.tls.client_ca_file"); ve == nil { + t.Fatal("unparseable client CA file was accepted") + } +} + +func TestValidate_DefaultConfigHasTLSOff(t *testing.T) { + cfg := config.DefaultConfig() + if cfg.HTTP.TLS.Enabled { + t.Error("http.tls.enabled defaults to true, want false") + } + if cfg.NATS.TLS.Enabled { + t.Error("nats.tls.enabled defaults to true, want false") + } + if errs := config.Validate(cfg); len(errs) != 0 { + t.Errorf("default config does not validate: %v", errs) + } +} + +// TestLoad_TLSFromFileAndEnv covers the loader wiring for all seven new keys. +// +// Validation tests exercise the structs directly, so nothing else here would +// notice a wrong `yaml:` tag, a key omitted from fileConfig, or a missing env +// binding — the value would simply never arrive and TLS would stay off with no +// error. nats.tls.client_ca_file in particular has no other coverage in any +// suite. +func TestLoad_TLSFromFileAndEnv(t *testing.T) { + cert, key, ca := writeCerts(t, time.Hour) + path := filepath.Join(t.TempDir(), "sqi.yaml") + body := "http:\n tls:\n enabled: true\n cert_file: " + cert + "\n key_file: " + key + + "\nnats:\n tls:\n enabled: true\n cert_file: " + cert + "\n key_file: " + key + + "\n client_ca_file: " + ca + "\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := config.Load(path, config.FlagOverrides{}) + if err != nil { + t.Fatalf("Load: %v", err) + } + if !cfg.HTTP.TLS.Enabled || cfg.HTTP.TLS.CertFile != cert || cfg.HTTP.TLS.KeyFile != key { + t.Errorf("http.tls from file = %+v, want enabled with %s/%s", cfg.HTTP.TLS, cert, key) + } + if !cfg.NATS.TLS.Enabled || cfg.NATS.TLS.CertFile != cert || cfg.NATS.TLS.KeyFile != key { + t.Errorf("nats.tls from file = %+v, want enabled with %s/%s", cfg.NATS.TLS, cert, key) + } + if cfg.NATS.TLS.ClientCAFile != ca { + t.Errorf("nats.tls.client_ca_file = %q, want %q", cfg.NATS.TLS.ClientCAFile, ca) + } + + // Env must override the file, for every key that has a binding. + t.Setenv("SQI_HTTP_TLS_ENABLED", "false") + t.Setenv("SQI_HTTP_TLS_CERT_FILE", "/env/http.crt") + t.Setenv("SQI_HTTP_TLS_KEY_FILE", "/env/http.key") + t.Setenv("SQI_NATS_TLS_ENABLED", "false") + t.Setenv("SQI_NATS_TLS_CERT_FILE", "/env/nats.crt") + t.Setenv("SQI_NATS_TLS_KEY_FILE", "/env/nats.key") + t.Setenv("SQI_NATS_TLS_CLIENT_CA_FILE", "/env/ca.pem") + + cfg, err = config.Load(path, config.FlagOverrides{}) + if err != nil { + t.Fatalf("Load with env: %v", err) + } + for _, tc := range []struct{ got, want, name string }{ + {cfg.HTTP.TLS.CertFile, "/env/http.crt", "SQI_HTTP_TLS_CERT_FILE"}, + {cfg.HTTP.TLS.KeyFile, "/env/http.key", "SQI_HTTP_TLS_KEY_FILE"}, + {cfg.NATS.TLS.CertFile, "/env/nats.crt", "SQI_NATS_TLS_CERT_FILE"}, + {cfg.NATS.TLS.KeyFile, "/env/nats.key", "SQI_NATS_TLS_KEY_FILE"}, + {cfg.NATS.TLS.ClientCAFile, "/env/ca.pem", "SQI_NATS_TLS_CLIENT_CA_FILE"}, + } { + if tc.got != tc.want { + t.Errorf("%s did not override the config file: got %q, want %q", tc.name, tc.got, tc.want) + } + } + if cfg.HTTP.TLS.Enabled { + t.Error("SQI_HTTP_TLS_ENABLED=false did not override the config file") + } + if cfg.NATS.TLS.Enabled { + t.Error("SQI_NATS_TLS_ENABLED=false did not override the config file") + } +} diff --git a/internal/config/validate.go b/internal/config/validate.go index a938ef75..92ef93e9 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -62,7 +62,8 @@ func validateHTTP(cfg HTTPConfig) []ValidationError { Message: fmt.Sprintf("invalid address %q: %s", cfg.Addr, err), }} } - return validateCORSOrigins(cfg.CORSOrigins) + errs := validateCORSOrigins(cfg.CORSOrigins) + return append(errs, validateHTTPTLS(cfg.TLS)...) } // validateCORSOrigins rejects origins go-chi/cors can never match, so a typo @@ -135,6 +136,11 @@ func validateNATS(cfg NATSConfig) []ValidationError { Message: fmt.Sprintf("must be > 0, got %d; set SQI_NATS_MAX_STORE_MB or nats.max_store_mb", cfg.MaxStoreMB), }) } + // TLS is validated BEFORE the auth early-return below: the two blocks are + // independent, and a farm running TLS with broker authentication off must + // still have its certificates checked at load. + errs = append(errs, validateNATSTLS(cfg.TLS)...) + // A disabled auth block must never produce validation errors — the same // rule validateAuth follows. Turning a feature off must not stop a // server from starting because of a value nobody is reading. diff --git a/internal/discovery/responder.go b/internal/discovery/responder.go index e1522c52..0729aa6c 100644 --- a/internal/discovery/responder.go +++ b/internal/discovery/responder.go @@ -65,6 +65,35 @@ type Config struct { // InstanceID uniquely identifies this server process. When empty, [New] // generates a random UUID. Advertised in the "id" TXT record. InstanceID string + + // HTTPTLS reports whether the REST/WebSocket listener is TLS-terminated, + // advertised as the "tls" TXT record. + // + // Consumed by internal/worker/discovery, which surfaces it as + // Result.HTTPTLS. Enrollment never uses a discovered URL, so a worker uses + // this to warn that its http:// nats.server_url points at an HTTPS server + // rather than to rewrite anything. + HTTPTLS bool + + // Interfaces restricts which network interfaces the service is advertised + // on. Nil — the production value — advertises on every multicast-capable + // interface, which is the point of mDNS discovery. + // + // It is set by tests, to loopback, so a test run never transmits an + // advertisement onto the network it happens to be running on. It has no + // config key: an operator who wants to limit advertisement should say so + // through a real setting, and none exists yet. + Interfaces []net.Interface + + // NATSTLS reports whether the embedded broker requires TLS, advertised as + // the "nats_tls" TXT record. + // + // This is the one signal that can tell a worker its broker needs TLS + // without anything being configured locally, so it feeds + // nats.tls_enabled's "auto" mode directly: a discovering worker turns TLS + // on rather than attempting plaintext and failing with an error that names + // neither the cause nor the fix. + NATSTLS bool } // Responder owns the lifecycle of the mDNS service advertisement. Create one @@ -127,7 +156,7 @@ func (r *Responder) Start(ctx context.Context) error { return nil } - txt := buildTXTRecords(r.cfg.InstanceID, r.httpPort, r.natsPort, hostname(), version.Version) + txt := buildTXTRecords(r.cfg.InstanceID, r.httpPort, r.natsPort, hostname(), version.Version, r.cfg.HTTPTLS, r.cfg.NATSTLS) server, err := zeroconf.Register( r.cfg.InstanceName, @@ -135,7 +164,7 @@ func (r *Responder) Start(ctx context.Context) error { domain, r.httpPort, txt, - nil, // nil interfaces = advertise on all multicast-capable interfaces + r.cfg.Interfaces, // nil = every multicast-capable interface ) if err != nil { return fmt.Errorf("discovery: register mDNS service: %w", err) @@ -189,14 +218,25 @@ func portFromAddr(addr string) (int, error) { // buildTXTRecords assembles the DNS-SD TXT key=value records advertised // alongside the service. Order is stable for deterministic tests. -func buildTXTRecords(instanceID string, httpPort, natsPort int, host, ver string) []string { - return []string{ +func buildTXTRecords(instanceID string, httpPort, natsPort int, host, ver string, httpTLS, natsTLS bool) []string { + txt := []string{ "id=" + instanceID, "http=" + strconv.Itoa(httpPort), "nats=" + strconv.Itoa(natsPort), "host=" + host, "version=" + ver, } + // The TLS keys are OMITTED rather than emitted as "tls=0" when off, so a + // plaintext farm advertises byte-for-byte what it always has. Clients that + // predate these keys ignore unknown keys, so adding them is compatible in + // both directions. + if httpTLS { + txt = append(txt, "tls=1") + } + if natsTLS { + txt = append(txt, "nats_tls=1") + } + return txt } // hostname returns the host's reported name, or "unknown" if it cannot be diff --git a/internal/discovery/responder_test.go b/internal/discovery/responder_test.go index 72edd15e..c7b3cd4b 100644 --- a/internal/discovery/responder_test.go +++ b/internal/discovery/responder_test.go @@ -57,7 +57,7 @@ func TestPortFromAddr(t *testing.T) { func TestBuildTXTRecords(t *testing.T) { t.Parallel() - got := buildTXTRecords("abc-123", 8080, 4222, "render-01", "v0.1.0") + got := buildTXTRecords("abc-123", 8080, 4222, "render-01", "v0.1.0", false, false) want := []string{ "id=abc-123", "http=8080", @@ -70,6 +70,51 @@ func TestBuildTXTRecords(t *testing.T) { } } +// TestBuildTXTRecords_TLSKeysOmittedWhenOff pins that a plaintext farm's +// advertisement is byte-for-byte what it has always been: the TLS keys are +// ABSENT, not present-and-zero. Anything else would change what every +// existing client sees on an unchanged deployment. +func TestBuildTXTRecords_TLSKeysOmittedWhenOff(t *testing.T) { + t.Parallel() + + for _, rec := range buildTXTRecords("id-1", 8080, 4222, "host", "0.4.0", false, false) { + if strings.HasPrefix(rec, "tls=") || strings.HasPrefix(rec, "nats_tls=") { + t.Errorf("plaintext advertisement contains %q; TLS keys must be absent, not zero-valued", rec) + } + } +} + +func TestBuildTXTRecords_TLSKeysPresentWhenOn(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + httpTLS, natsTLS bool + want []string + absent []string + }{ + {"both on", true, true, []string{"tls=1", "nats_tls=1"}, nil}, + {"http only", true, false, []string{"tls=1"}, []string{"nats_tls=1"}}, + {"nats only", false, true, []string{"nats_tls=1"}, []string{"tls=1"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := buildTXTRecords("id-1", 8080, 4222, "host", "0.4.0", tt.httpTLS, tt.natsTLS) + for _, w := range tt.want { + if !slices.Contains(got, w) { + t.Errorf("records %v missing %q", got, w) + } + } + for _, a := range tt.absent { + if slices.Contains(got, a) { + t.Errorf("records %v unexpectedly contain %q", got, a) + } + } + }) + } +} + func TestNew_Disabled(t *testing.T) { t.Parallel() diff --git a/internal/server/plaintextdefault_test.go b/internal/server/plaintextdefault_test.go new file mode 100644 index 00000000..8a0d3505 --- /dev/null +++ b/internal/server/plaintextdefault_test.go @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package server + +import ( + "crypto/tls" + "net/http" + "strings" + "testing" + "time" + + nats "github.com/nats-io/nats.go" + + "github.com/uberware/sqi/internal/config" +) + +// TestDefaultConfig_ServesPlaintextUnchanged is Phase 4's named +// default-configuration regression test for H2. +// +// The phase's standing rule is that the single-binary, SQLite, embedded-NATS, +// no-TLS deployment must keep working exactly as it did in v0.3.0. Every other +// test in this component adds a TLS path; this one proves the absence of one. +// Each assertion states a property of the plaintext default that a future TLS +// change would break loudly rather than quietly. +func TestDefaultConfig_ServesPlaintextUnchanged(t *testing.T) { + // Derived from config.DefaultConfig() rather than hand-written, so a + // default that starts shipping TLS on would fail here instead of being + // invisible to a literal struct. + defaults := config.DefaultConfig() + if defaults.HTTP.TLS.Enabled { + t.Fatal("config.DefaultConfig() ships http.tls.enabled = true; TLS must be opt-in") + } + if defaults.NATS.TLS.Enabled { + t.Fatal("config.DefaultConfig() ships nats.tls.enabled = true; TLS must be opt-in") + } + + // Same boot helper the TLS tests use, so the only difference between this + // test and those is the TLS configuration itself. + srv, base := startTestServer(t, "http", func(cfg *Config) { + cfg.HTTPTLS = defaults.HTTP.TLS + cfg.NATSTLS = defaults.NATS.TLS + }) + httpAddr := strings.TrimPrefix(base, "http://") + natsAddr := srv.cfg.NATSAddr + + client := &http.Client{Timeout: 5 * time.Second} + + // 1. Plain HTTP serves. + deadline := time.Now().Add(10 * time.Second) + var resp *http.Response + var err error + for time.Now().Before(deadline) { + resp, err = get(t, client, base+"/healthz") + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + if err != nil { + t.Fatalf("GET /healthz over plain HTTP: %v", err) + } + code := resp.StatusCode + _ = resp.Body.Close() + if code != http.StatusOK { + t.Errorf("/healthz status = %d, want 200", code) + } + + // 2. There is no TLS listener on that port. + tlsClient := &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + // InsecureSkipVerify is deliberate: the assertion is that no TLS + // listener exists at all, so verification must not be what fails. + TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}, + }, + } + if r, err := get(t, tlsClient, "https://"+httpAddr+"/healthz"); err == nil { + _ = r.Body.Close() + t.Error("an HTTPS request succeeded against a default-configured server") + } + + // 3. The http.Server carries no CERTIFICATE. + // + // Deliberately not "TLSConfig == nil": net/http's HTTP/2 setup builds a + // TLSConfig with NextProtos ["h2","http/1.1"] even for a plain + // ListenAndServe, so a nil check would assert an implementation detail + // of net/http rather than anything about sqi. What must hold is that no + // certificate was loaded — that is what would make the listener speak + // TLS. + if tc := srv.httpServer.TLSConfig; tc != nil && len(tc.Certificates) > 0 { + t.Errorf("httpServer.TLSConfig carries %d certificate(s) on a default-configured server", len(tc.Certificates)) + } + + // 4. The broker is plaintext: a client with no TLS options connects. + nc, err := nats.Connect("nats://"+natsAddr, nats.Timeout(5*time.Second)) + if err != nil { + t.Fatalf("plaintext NATS client could not connect to a default broker: %v", err) + } + nc.Close() + + // 5. Session cookies: NOT asserted here. + // + // The default is cookie_secure "auto", which resolves from r.TLS, so on + // this plaintext listener a cookie must not be Secure. Checking the + // default string here would assert a constant, not a behavior; + // TestSessionCookie_NotSecureOnPlaintext in internal/api drives an actual + // login over an actual plaintext listener and inspects the real cookie. + + // 6. The mDNS advertisement carries no TLS keys. + // + // Asserted where the records are actually built, not here: this server + // runs with DiscoveryEnabled=false (multicast is unavailable in most CI + // environments), so a responder started here would never produce + // records and any check would be vacuous. + // TestBuildTXTRecords_TLSKeysOmittedWhenOff in internal/discovery is the + // real assertion. What this test contributes is the input to it: both + // flags come from cfg.HTTPTLS.Enabled / cfg.NATSTLS.Enabled, verified + // false at the top of this function. +} diff --git a/internal/server/revokeworker_test.go b/internal/server/revokeworker_test.go index 61081a61..c564af1e 100644 --- a/internal/server/revokeworker_test.go +++ b/internal/server/revokeworker_test.go @@ -51,14 +51,18 @@ func freeLoopbackAddr(t *testing.T) string { // startTestBroker boots a real embedded broker with authentication enabled // and enrolled with creds, and registers cleanup. -func startTestBroker(t *testing.T, creds []bus.WorkerCredentialRef) *bus.Broker { +func startTestBroker(t *testing.T, creds []bus.WorkerCredentialRef, mutate ...func(*bus.BrokerConfig)) *bus.Broker { t.Helper() - b := bus.New(bus.BrokerConfig{ + cfg := bus.BrokerConfig{ Addr: freeLoopbackAddr(t), DataDir: t.TempDir() + "/nats", MaxStoreMB: 64, Auth: bus.BrokerAuthConfig{Enabled: true, Credentials: creds}, - }, testLogger()) + } + for _, m := range mutate { + m(&cfg) + } + b := bus.New(cfg, testLogger()) ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() if err := b.Start(ctx); err != nil { diff --git a/internal/server/server.go b/internal/server/server.go index 712ca3e2..80a1bfe1 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -42,6 +42,7 @@ import ( "github.com/uberware/sqi/internal/scheduler" "github.com/uberware/sqi/internal/store" "github.com/uberware/sqi/internal/store/sqlite" + "github.com/uberware/sqi/internal/tlsutil" "github.com/uberware/sqi/internal/version" "github.com/uberware/sqi/internal/ws" ) @@ -60,6 +61,14 @@ type Config struct { // HTTPAddr is the TCP address the REST + WebSocket server listens on. HTTPAddr string // default "0.0.0.0:8080" + // HTTPTLS terminates HTTPS in-process on the REST/WebSocket listener. + // Zero value leaves it plaintext, which is the default. + HTTPTLS config.TLSConfig + + // NATSTLS encrypts the embedded broker's client listener. Zero value + // leaves it plaintext, which is the default. + NATSTLS config.NATSTLSConfig + // CORSOrigins is the list of origins that the CORS middleware permits. // Use ["*"] to allow all origins (suitable for local / dev deployments). // An empty slice is treated as ["*"]. Tighten this for production @@ -129,6 +138,11 @@ type Config struct { // multicast (most cloud VPCs). Default true. DiscoveryEnabled bool + // DiscoveryInterfaces restricts mDNS advertisement to specific interfaces. + // Nil — the production value — advertises on all multicast-capable ones. + // Set by tests to loopback; see internal/discovery.Config.Interfaces. + DiscoveryInterfaces []net.Interface + // DiscoveryInstanceName is the mDNS service instance name advertised on // the network. Each server on the same subnet should use a distinct name. // Default "sqi-server". @@ -307,6 +321,81 @@ func (s *Server) Metrics() *metrics.Metrics { return s.metrics } +// serveHTTP configures TLS on s.httpServer when it is enabled and starts the +// listener in a background goroutine. Split out of [Server.start] to keep that +// function under the cyclop complexity threshold. +func (s *Server) serveHTTP(ctx context.Context) error { + // Both TLS surfaces are warned about here, beside the listener they concern, + // rather than riding on broker startup where a reordering would silently + // take the HTTP warning with it. + warnTLSConfig(ctx, s.logger, s.cfg.HTTPTLS, s.cfg.NATSTLS) + + tlsOn := s.cfg.HTTPTLS.Enabled + if tlsOn { + tlsCfg, err := tlsutil.ServerConfig(s.cfg.HTTPTLS.CertFile, s.cfg.HTTPTLS.KeyFile, "") + if err != nil { + return fmt.Errorf("http: tls: %w", err) + } + s.httpServer.TLSConfig = tlsCfg + } + + go func() { + s.logger.InfoContext(ctx, "http: listening", + slog.String("addr", s.cfg.HTTPAddr), + slog.Bool("tls", tlsOn), + slog.String("url", browseURL(s.cfg.HTTPAddr, tlsOn))) + + // Certificates are already loaded into TLSConfig, so both paths pass + // empty filenames: re-reading them here could pick up a half-written + // file if a rotation lands between validation and listen. + var err error + if tlsOn { + err = s.httpServer.ListenAndServeTLS("", "") + } else { + err = s.httpServer.ListenAndServe() + } + if err != nil && !errors.Is(err, http.ErrServerClosed) { + s.logger.ErrorContext(ctx, "http: server error", slog.Any("error", err)) + } + }() + return nil +} + +// warnTLSConfig emits the advisory startup warnings for both TLS blocks: +// certificates staged but not switched on, and certificates close to expiry. +// +// Neither is a load error — staging a certificate before flipping the switch +// is a legitimate rollout step, and a certificate 29 days from expiry still +// works — but both are worth saying out loud once at startup. The broker's +// warnings are emitted here rather than in internal/bus so that package does +// not have to import internal/config. +func warnTLSConfig(ctx context.Context, logger *slog.Logger, httpTLS config.TLSConfig, natsTLS config.NATSTLSConfig) { + surfaces := []struct { + component string // log prefix + key string // the config key an operator would flip + enabled bool + certFile string + keyFile string + }{ + {"http", "http.tls.enabled", httpTLS.Enabled, httpTLS.CertFile, httpTLS.KeyFile}, + {"bus", "nats.tls.enabled", natsTLS.Enabled, natsTLS.CertFile, natsTLS.KeyFile}, + } + for _, s := range surfaces { + if !s.enabled { + if s.certFile != "" || s.keyFile != "" { + logger.WarnContext(ctx, + s.component+": tls certificates are configured but "+s.key+" is false; serving plaintext", + slog.String("cert_file", s.certFile)) + } + continue + } + if notAfter, soon := config.ExpiringSoon(s.certFile); soon { + logger.WarnContext(ctx, s.component+": tls certificate expires soon", + slog.String("cert_file", s.certFile), slog.Time("not_after", notAfter)) + } + } +} + // warnIfBrokerUnauthenticated emits a WARN when the NATS broker is reachable // from outside this machine and has no credential requirement. // @@ -364,6 +453,15 @@ func (s *Server) startBroker(ctx context.Context) (*bus.Broker, error) { DataDir: s.cfg.NATSDataDir, MaxStoreMB: s.cfg.NATSMaxStoreMB, Auth: brokerAuth, + // Mapped field by field rather than shared: internal/bus does not + // import internal/config, the same boundary WorkerCredentialRef draws + // against internal/store. + TLS: bus.BrokerTLSConfig{ + Enabled: s.cfg.NATSTLS.Enabled, + CertFile: s.cfg.NATSTLS.CertFile, + KeyFile: s.cfg.NATSTLS.KeyFile, + ClientCAFile: s.cfg.NATSTLS.ClientCAFile, + }, }, s.logger) if err := broker.Start(ctx); err != nil { return nil, err @@ -651,14 +749,9 @@ func (s *Server) start(ctx context.Context) error { ReadHeaderTimeout: 10 * time.Second, } - go func() { - s.logger.InfoContext(ctx, "http: listening", - slog.String("addr", s.cfg.HTTPAddr), - slog.String("url", browseURL(s.cfg.HTTPAddr))) - if err := s.httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - s.logger.ErrorContext(ctx, "http: server error", slog.Any("error", err)) - } - }() + if err := s.serveHTTP(ctx); err != nil { + return err + } // ── mDNS responder ──────────────────────────────────────────────────── // Advertise _sqi._tcp on the local network so workers and the @@ -670,6 +763,9 @@ func (s *Server) start(ctx context.Context) error { InstanceName: s.cfg.DiscoveryInstanceName, HTTPAddr: s.cfg.HTTPAddr, NATSAddr: s.cfg.NATSAddr, + HTTPTLS: s.cfg.HTTPTLS.Enabled, + NATSTLS: s.cfg.NATSTLS.Enabled, + Interfaces: s.cfg.DiscoveryInterfaces, }, s.logger) if err != nil { return fmt.Errorf("init discovery: %w", err) @@ -939,17 +1035,21 @@ func toOIDCConfig(c config.OIDCConfig, logger *slog.Logger) oidc.Config { // any wildcard or empty host, leaving an explicit host (e.g. "127.0.0.1" or a // LAN IP) untouched. The original bind address is still logged separately under // the "addr" key for operators. -func browseURL(addr string) string { +func browseURL(addr string, tlsOn bool) string { + scheme := "http://" + if tlsOn { + scheme = "https://" + } host, port, err := net.SplitHostPort(addr) if err != nil { // Not in host:port form; surface it as-is rather than guessing. - return "http://" + addr + return scheme + addr } switch host { case "", "0.0.0.0", "::", "[::]": host = "localhost" } - return "http://" + net.JoinHostPort(host, port) + return scheme + net.JoinHostPort(host, port) } // shutdown stops all running components in reverse dependency order within diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 0c308112..73d85e3f 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -12,21 +12,26 @@ import ( func TestBrowseURL(t *testing.T) { tests := []struct { - name string - addr string - want string + name string + addr string + tlsOn bool + want string }{ - {"ipv4 wildcard becomes localhost", "0.0.0.0:8080", "http://localhost:8080"}, - {"ipv6 wildcard becomes localhost", "[::]:8080", "http://localhost:8080"}, - {"empty host becomes localhost", ":8080", "http://localhost:8080"}, - {"explicit loopback is preserved", "127.0.0.1:8080", "http://127.0.0.1:8080"}, - {"explicit lan ip is preserved", "192.168.1.10:9000", "http://192.168.1.10:9000"}, - {"non host:port surfaced as-is", "not-an-addr", "http://not-an-addr"}, + {"ipv4 wildcard becomes localhost", "0.0.0.0:8080", false, "http://localhost:8080"}, + {"ipv6 wildcard becomes localhost", "[::]:8080", false, "http://localhost:8080"}, + {"empty host becomes localhost", ":8080", false, "http://localhost:8080"}, + {"explicit loopback is preserved", "127.0.0.1:8080", false, "http://127.0.0.1:8080"}, + {"explicit lan ip is preserved", "192.168.1.10:9000", false, "http://192.168.1.10:9000"}, + {"non host:port surfaced as-is", "not-an-addr", false, "http://not-an-addr"}, + {"tls wildcard becomes https localhost", "0.0.0.0:8080", true, "https://localhost:8080"}, + {"tls explicit host keeps scheme", "sqi.example:8443", true, "https://sqi.example:8443"}, + {"tls ipv6 wildcard", "[::]:8080", true, "https://localhost:8080"}, + {"tls non host:port surfaced as-is", "not-an-addr", true, "https://not-an-addr"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := browseURL(tt.addr); got != tt.want { - t.Errorf("browseURL(%q) = %q, want %q", tt.addr, got, tt.want) + if got := browseURL(tt.addr, tt.tlsOn); got != tt.want { + t.Errorf("browseURL(%q, %v) = %q, want %q", tt.addr, tt.tlsOn, got, tt.want) } }) } diff --git a/internal/server/tls_test.go b/internal/server/tls_test.go new file mode 100644 index 00000000..5f08cbe3 --- /dev/null +++ b/internal/server/tls_test.go @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package server + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" + "github.com/uberware/sqi/internal/config" + "github.com/uberware/sqi/internal/scheduler" +) + +// writeServerCerts generates a farm CA and a server keypair covering +// localhost/127.0.0.1 into a temp dir. validFor is threaded through so a +// caller can mint a certificate that is expiring soon, or already expired. +func writeServerCerts(t *testing.T, validFor time.Duration) (certFile, keyFile, caFile string) { + t.Helper() + dir := t.TempDir() + ca, err := certgen.NewCA("test CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"localhost", "127.0.0.1", "::1"}, validFor) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "server", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + return filepath.Join(dir, "server.crt"), filepath.Join(dir, "server.key"), filepath.Join(dir, "ca.crt") +} + +// get performs a GET and returns the response. It exists because the lint +// config forbids http.Client.Get (noctx) in favor of an explicit request. +func get(t *testing.T, client *http.Client, url string) (*http.Response, error) { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + t.Fatalf("build request for %s: %v", url, err) + } + return client.Do(req) +} + +// startTestServer boots a real Server on free loopback ports, applies mutate +// to its Config, waits for the listener, and returns the base URL. +// +// Shared by the TLS tests and by the plaintext-default regression test, so the +// two cannot drift apart in how they boot a server — the difference between +// them should be the TLS config and nothing else. +func startTestServer(t *testing.T, scheme string, mutate func(*Config)) (*Server, string) { + t.Helper() + httpAddr := freeLoopbackAddr(t) + natsAddr := freeLoopbackAddr(t) + tmpDir := t.TempDir() + + cfg := Config{ + HTTPAddr: httpAddr, + CORSOrigins: []string{"*"}, + NATSAddr: natsAddr, + NATSDataDir: tmpDir + "/nats", + NATSMaxStoreMB: 64, + SQLitePath: tmpDir + "/test.db", + CheckpointInterval: time.Minute, + Scheduler: scheduler.Config{ + AssignInterval: 100 * time.Millisecond, + AssignBatchSize: 10, + AssignWorkers: 2, + WorkerTimeout: 30 * time.Second, + HeartbeatSweepInterval: 15 * time.Second, + }, + DiscoveryEnabled: false, + } + if mutate != nil { + mutate(&cfg) + } + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + srv := New(cfg, logger, nil) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- srv.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Error("server did not shut down within 30s") + } + }) + + select { + case err := <-done: + t.Fatalf("server exited during startup: %v", err) + case <-time.After(50 * time.Millisecond): + } + + dialer := &net.Dialer{Timeout: 200 * time.Millisecond} + deadline := time.Now().Add(10 * time.Second) + for !time.Now().After(deadline) { + c, err := dialer.DialContext(context.Background(), "tcp", httpAddr) + if err == nil { + _ = c.Close() + return srv, scheme + "://" + httpAddr + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("listener did not come up on %s within 10s", httpAddr) + return nil, "" +} + +// startTLSServer boots a server with TLS enabled and returns its https:// URL. +func startTLSServer(t *testing.T, certFile, keyFile string) string { + t.Helper() + _, base := startTestServer(t, "https", func(cfg *Config) { + cfg.HTTPTLS = config.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + }) + return base +} + +// caPool returns a root pool trusting only caFile. +func caPool(t *testing.T, caFile string) *x509.CertPool { + t.Helper() + pem, err := os.ReadFile(caFile) + if err != nil { + t.Fatalf("read CA: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("CA did not append to pool") + } + return pool +} + +// caClient returns an HTTP client trusting only caFile. +func caClient(t *testing.T, caFile string) *http.Client { + t.Helper() + pool := caPool(t, caFile) + return &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }, + } +} + +func TestServer_ServesHTTPSWithFarmCA(t *testing.T) { + certFile, keyFile, caFile := writeServerCerts(t, 365*24*time.Hour) + base := startTLSServer(t, certFile, keyFile) + + resp, err := get(t, caClient(t, caFile), base+"/healthz") + if err != nil { + t.Fatalf("GET %s/healthz over TLS: %v", base, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } +} + +func TestServer_HTTPSRejectedByUntrustingClient(t *testing.T) { + certFile, keyFile, _ := writeServerCerts(t, 365*24*time.Hour) + base := startTLSServer(t, certFile, keyFile) + + // System roots only: the farm CA is not among them, so this must fail. + client := &http.Client{Timeout: 5 * time.Second} + resp, err := get(t, client, base+"/healthz") + if err == nil { + _ = resp.Body.Close() + t.Fatal("request succeeded with the system root pool; the server is not presenting the farm certificate") + } + // The failure must be certificate verification, not a transport error that + // would also "pass" this test for the wrong reason. + if !strings.Contains(err.Error(), "certificate") { + t.Errorf("error = %v, want a certificate verification failure", err) + } +} + +func TestServer_PlaintextRequestToTLSPortIsRefusedNotHung(t *testing.T) { + certFile, keyFile, _ := writeServerCerts(t, 365*24*time.Hour) + base := startTLSServer(t, certFile, keyFile) + + // Same address, http:// scheme. Go's server answers this with a 400 and a + // readable body rather than hanging; the timeout makes a regression to a + // hang fail fast instead of stalling CI. + plain := "http://" + base[len("https://"):] + client := &http.Client{Timeout: 5 * time.Second} + resp, err := get(t, client, plain+"/healthz") + if err != nil { + t.Fatalf("plaintext request to the TLS port errored instead of returning a status: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("status = %d, want 400", resp.StatusCode) + } +} + +func TestServer_ReadyzUnderTLSInsideHealthBudget(t *testing.T) { + certFile, keyFile, caFile := writeServerCerts(t, 365*24*time.Hour) + base := startTLSServer(t, certFile, keyFile) + client := caClient(t, caFile) + + // /readyz gates on SQLite and NATS. internal/health budgets 5s; the TLS + // handshake must not push it over. + deadline := time.Now().Add(15 * time.Second) + for { + start := time.Now() + resp, err := get(t, client, base+"/readyz") + elapsed := time.Since(start) + if err == nil { + code := resp.StatusCode + _ = resp.Body.Close() + if code == http.StatusOK { + if elapsed >= 5*time.Second { + t.Errorf("/readyz took %s over TLS, want well inside the 5s health budget", elapsed) + } + return + } + } + if time.Now().After(deadline) { + t.Fatalf("/readyz never returned 200 over TLS within 15s (last err: %v)", err) + } + time.Sleep(100 * time.Millisecond) + } +} + +func TestTLSStartupWarnings(t *testing.T) { + expiringCert, expiringKey, _ := writeServerCerts(t, 10*24*time.Hour) // inside the 30-day window + goodCert, goodKey, _ := writeServerCerts(t, 365*24*time.Hour) + + tests := []struct { + name string + httpTLS config.TLSConfig + natsTLS config.NATSTLSConfig + wantWarn string + }{ + { + name: "http certificates configured but tls disabled", + httpTLS: config.TLSConfig{Enabled: false, CertFile: goodCert, KeyFile: goodKey}, + wantWarn: "http.tls.enabled", + }, + { + name: "nats certificates configured but tls disabled", + natsTLS: config.NATSTLSConfig{Enabled: false, CertFile: goodCert, KeyFile: goodKey}, + wantWarn: "nats.tls.enabled", + }, + { + name: "http certificate expiring soon", + httpTLS: config.TLSConfig{Enabled: true, CertFile: expiringCert, KeyFile: expiringKey}, + wantWarn: "expires soon", + }, + { + name: "nats certificate expiring soon", + natsTLS: config.NATSTLSConfig{Enabled: true, CertFile: expiringCert, KeyFile: expiringKey}, + wantWarn: "expires soon", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn})) + warnTLSConfig(context.Background(), logger, tt.httpTLS, tt.natsTLS) + if !strings.Contains(buf.String(), tt.wantWarn) { + t.Errorf("warnings = %q, want one containing %q", buf.String(), tt.wantWarn) + } + }) + } +} + +func TestTLSStartupWarnings_SilentOnDefaults(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn})) + warnTLSConfig(context.Background(), logger, config.TLSConfig{}, config.NATSTLSConfig{}) + if buf.Len() != 0 { + t.Errorf("default configuration produced warnings: %q", buf.String()) + } +} diff --git a/internal/server/tlsreload_test.go b/internal/server/tlsreload_test.go new file mode 100644 index 00000000..1edea609 --- /dev/null +++ b/internal/server/tlsreload_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package server + +import ( + "context" + "crypto/tls" + "strings" + "testing" + "time" + + nats "github.com/nats-io/nats.go" + + "github.com/uberware/sqi/internal/bus" + "github.com/uberware/sqi/internal/store/fake" +) + +// TestRevokeWorker_TLSSurvivesCredentialReload pins the one behavior that +// lives only at the seam between H1 and H2, and that neither component's own +// tests would ever exercise. +// +// RevokeWorker calls bus.Broker.ReloadCredentials, which clones the pristine +// boot Options and hands them to nats-server's ReloadOptions. Options.Clone() +// deep-copies TLSConfig (nats-server opts.go:828), so TLS survives. If that +// ever stopped being true — or if broker.go set opts.TLSConfig AFTER taking +// the bootOpts clone — then revoking any single worker would silently drop +// the entire broker to plaintext, and every other test in the tree would +// still pass. +func TestRevokeWorker_TLSSurvivesCredentialReload(t *testing.T) { + certFile, keyFile, caFile := writeServerCerts(t, 365*24*time.Hour) + st := fake.New() + refA, seedA := enrolledCredential(t, st, "worker-a") + refB, _ := enrolledCredential(t, st, "worker-b") + + broker := startTestBroker(t, []bus.WorkerCredentialRef{refA, refB}, func(c *bus.BrokerConfig) { + c.TLS = bus.BrokerTLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + }) + s := &Server{cfg: Config{NATSAuthEnabled: true}, store: st, broker: broker, logger: testLogger()} + pool := caPool(t, caFile) + + secure := nats.Secure(&tls.Config{MinVersion: tls.VersionTLS12, RootCAs: pool}) + + // A connects over TLS before the reload. + ncA, err := nats.Connect(broker.ClientURL(), secure, nkeyOption(t, seedA, refA.PublicKey), nats.NoReconnect()) + if err != nil { + t.Fatalf("worker A could not connect over TLS: %v", err) + } + ncA.Close() + + // Revoke B. This is the call that reloads the broker's options. + if err := s.RevokeWorker(context.Background(), refB.WorkerID); err != nil { + t.Fatalf("RevokeWorker: %v", err) + } + + // THE assertion: TLS is still required after the reload. + // + // The plaintext client carries credential A's VALID nkey. Without it the + // broker refuses the connection for want of authentication whatever the TLS + // state, and the check cannot tell "TLS still required" from "auth still + // required" — it passed even with the ordering regression injected. With a + // valid credential, refusal can only mean TLS. + plain, err := nats.Connect(broker.ClientURL(), nats.NoReconnect(), + nkeyOption(t, seedA, refA.PublicKey)) + if err == nil { + plain.Close() + t.Fatal("a plaintext client with a valid credential connected after a credential reload; the reload dropped broker TLS") + } + if !strings.Contains(err.Error(), "secure") && !strings.Contains(err.Error(), "TLS") && !strings.Contains(err.Error(), "tls") { + t.Errorf("plaintext client was refused for the wrong reason: %v — this assertion only means anything if TLS is what refused it", err) + } + + // And A can still connect over TLS afterwards. + ncA2, err := nats.Connect(broker.ClientURL(), secure, nkeyOption(t, seedA, refA.PublicKey), nats.NoReconnect()) + if err != nil { + t.Fatalf("worker A could not reconnect over TLS after the reload: %v", err) + } + ncA2.Close() +} diff --git a/internal/tlsutil/tlsutil.go b/internal/tlsutil/tlsutil.go new file mode 100644 index 00000000..e33e7c39 --- /dev/null +++ b/internal/tlsutil/tlsutil.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package tlsutil holds the TLS loading primitives shared by every component +// that terminates or dials TLS: the API listener, the embedded broker, the +// worker's broker client and the worker's enrollment client. +// +// It exists because those callers live on both sides of an import boundary. +// internal/config pulls internal/auth/oidc, so the worker binary can never +// import it, and internal/bus is forbidden from importing internal/config at +// all. This package imports nothing internal, so all of them can reach it. +// +// It deliberately holds only LOADING. Validation of operator-supplied paths +// stays in internal/config, which is the one place that knows config keys and +// can name them in an error. +package tlsutil + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" +) + +// CertPool reads a PEM bundle into a certificate pool. +// +// Callers use this for the "empty means system roots, set means THAT CA only" +// rule: an empty path never reaches here, and a non-empty one produces a pool +// that pins the acceptable issuer rather than widening it. +func CertPool(caFile string) (*x509.CertPool, error) { + pem, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read CA file %s: %w", caFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("no valid certificates found in CA file %s", caFile) + } + return pool, nil +} + +// ServerConfig builds a server *tls.Config from a certificate/key pair, +// optionally requiring client certificates signed by clientCAFile. +// +// It is called once at startup, with paths the caller has already validated, +// so the certificate is read exactly once: handing the paths to +// ListenAndServeTLS instead would read them a second time, and a rotation +// landing between the two reads would serve a half-written file. +func ServerConfig(certFile, keyFile, clientCAFile string) (*tls.Config, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("load keypair %s/%s: %w", certFile, keyFile, err) + } + cfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + } + if clientCAFile == "" { + return cfg, nil + } + pool, err := CertPool(clientCAFile) + if err != nil { + return nil, err + } + cfg.ClientCAs = pool + cfg.ClientAuth = tls.RequireAndVerifyClientCert + return cfg, nil +} + +// LeafOf returns the parsed leaf certificate of a loaded keypair. +// +// tls.LoadX509KeyPair normally populates Leaf itself, but only when the +// x509keypairleaf GODEBUG is left at its default; under +// GODEBUG=x509keypairleaf=0 it stays nil, so the re-parse below is a real +// fallback rather than dead code. +func LeafOf(pair tls.Certificate) (*x509.Certificate, error) { + if pair.Leaf != nil { + return pair.Leaf, nil + } + if len(pair.Certificate) == 0 { + return nil, errors.New("keypair contains no certificate") + } + return x509.ParseCertificate(pair.Certificate[0]) +} diff --git a/internal/tlsutil/tlsutil_test.go b/internal/tlsutil/tlsutil_test.go new file mode 100644 index 00000000..2a8db6bd --- /dev/null +++ b/internal/tlsutil/tlsutil_test.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package tlsutil_test + +import ( + "crypto/tls" + "os" + "path/filepath" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" + "github.com/uberware/sqi/internal/tlsutil" +) + +// farmCerts generates a CA and a server keypair into a temp dir and returns +// (certFile, keyFile, caFile). +func farmCerts(t *testing.T) (certFile, keyFile, caFile string) { + t.Helper() + dir := t.TempDir() + ca, err := certgen.NewCA("test CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"localhost", "127.0.0.1"}, time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "server", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + return filepath.Join(dir, "server.crt"), filepath.Join(dir, "server.key"), filepath.Join(dir, "ca.crt") +} + +func TestServerConfig_NoClientCAMeansNoClientAuth(t *testing.T) { + cert, key, _ := farmCerts(t) + tc, err := tlsutil.ServerConfig(cert, key, "") + if err != nil { + t.Fatalf("ServerConfig: %v", err) + } + if tc.ClientAuth != tls.NoClientCert { + t.Errorf("ClientAuth = %v, want NoClientCert", tc.ClientAuth) + } + if tc.MinVersion != tls.VersionTLS12 { + t.Errorf("MinVersion = %x, want TLS 1.2", tc.MinVersion) + } +} + +func TestServerConfig_ClientCARequiresAndVerifies(t *testing.T) { + cert, key, caFile := farmCerts(t) + tc, err := tlsutil.ServerConfig(cert, key, caFile) + if err != nil { + t.Fatalf("ServerConfig: %v", err) + } + if tc.ClientAuth != tls.RequireAndVerifyClientCert { + t.Errorf("ClientAuth = %v, want RequireAndVerifyClientCert", tc.ClientAuth) + } + if tc.ClientCAs == nil { + t.Error("ClientCAs is nil, want the configured CA pool") + } +} + +func TestServerConfig_BadPathIsReported(t *testing.T) { + cert, key, _ := farmCerts(t) + if _, err := tlsutil.ServerConfig(cert+".nope", key, ""); err == nil { + t.Error("a missing certificate file was accepted") + } +} + +func TestCertPool_RejectsUnparseableBundle(t *testing.T) { + garbage := filepath.Join(t.TempDir(), "garbage.pem") + if err := os.WriteFile(garbage, []byte("not a certificate\n"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + if _, err := tlsutil.CertPool(garbage); err == nil { + t.Error("an unparseable CA bundle was accepted") + } + if _, err := tlsutil.CertPool(filepath.Join(t.TempDir(), "absent.pem")); err == nil { + t.Error("a missing CA file was accepted") + } +} + +func TestLeafOf_ReturnsTheLeaf(t *testing.T) { + cert, key, _ := farmCerts(t) + pair, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + t.Fatalf("LoadX509KeyPair: %v", err) + } + leaf, err := tlsutil.LeafOf(pair) + if err != nil { + t.Fatalf("LeafOf: %v", err) + } + if leaf.NotAfter.IsZero() { + t.Error("leaf has a zero NotAfter") + } + + // The GODEBUG=x509keypairleaf=0 path: Leaf nil, re-parsed from DER. + pair.Leaf = nil + reparsed, err := tlsutil.LeafOf(pair) + if err != nil { + t.Fatalf("LeafOf with a nil Leaf: %v", err) + } + if !reparsed.NotAfter.Equal(leaf.NotAfter) { + t.Error("re-parsed leaf disagrees with the populated one") + } +} diff --git a/internal/worker/config/config.go b/internal/worker/config/config.go index 2492011a..7eec8495 100644 --- a/internal/worker/config/config.go +++ b/internal/worker/config/config.go @@ -14,6 +14,7 @@ package config import ( + "crypto/tls" "fmt" "os" "path/filepath" @@ -271,6 +272,23 @@ type NATSConfig struct { // Env: SQI_WORKER_NATS_URL URL string `yaml:"url"` + // TLSEnabled controls TLS on the broker connection. It is a THREE-VALUED + // string, not a bool, mirroring auth.session.cookie_secure on the server: + // + // - "auto" (default): use TLS when there is a reason to. That means any + // of the TLS fields below being set, or the discovered server + // advertising a TLS-required broker over mDNS. + // - "true": always use TLS, even with no CA configured (system roots) — + // for a broker presenting a publicly-trusted certificate, which + // nothing else would signal. + // - "false": never use TLS, even with a CA configured. An explicit + // override for an operator who knows the broker is plaintext. + // + // A bool cannot express this: the interesting states are "infer" and + // "force off", and a bool collapses them onto the same zero value. + // Env: SQI_WORKER_NATS_TLS_ENABLED + TLSEnabled string `yaml:"tls_enabled"` + // TLSCertFile is the path to the client TLS certificate (PEM). // Env: SQI_WORKER_NATS_TLS_CERT_FILE TLSCertFile string `yaml:"tls_cert_file"` @@ -326,6 +344,69 @@ type NATSConfig struct { // than attempting a request with no host. // Env: SQI_WORKER_NATS_SERVER_URL ServerURL string `yaml:"server_url"` + + // ServerTLSCAFile is the CA that verifies sqi-server's certificate at + // ServerURL, for enrollment over HTTPS. Empty means the system roots; + // set means THAT CA only. + // + // Deliberately separate from TLSCAFile above, which is documented as the + // NATS broker CA: the two can legitimately differ (a publicly-issued + // certificate on the API, a private farm CA on the broker), and reusing + // one key for both surfaces reads fine now and misleads later. + // Env: SQI_WORKER_NATS_SERVER_TLS_CA_FILE + ServerTLSCAFile string `yaml:"server_tls_ca_file"` + + // ServerTLSInsecureSkipVerify disables verification of sqi-server's + // certificate during enrollment. Development only. + // Env: SQI_WORKER_NATS_SERVER_TLS_INSECURE_SKIP_VERIFY + ServerTLSInsecureSkipVerify bool `yaml:"server_tls_insecure_skip_verify"` +} + +// TLS modes for [NATSConfig.TLSEnabled]. The zero value ("") is treated as +// TLSAuto so a struct built in a test behaves like a defaulted config. +const ( + TLSAuto = "auto" + TLSOn = "true" + TLSOff = "false" +) + +// normalizeTLSMode maps a configured tls_enabled value onto one of the three +// modes, or returns ok=false if it is not a value we understand. +// +// The truthy/falsy synonyms are accepted because SQI_WORKER_NATS_TLS_ENABLED=1 +// is how people set boolean-looking environment variables, and this key looked +// like a bool until recently. They are accepted in YAML too, so the file and +// the environment never disagree about what a value means. +func normalizeTLSMode(v string) (mode string, ok bool) { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", TLSAuto: + return TLSAuto, true + case TLSOn, "1", "yes", "on": + return TLSOn, true + case TLSOff, "0", "no", "off": + return TLSOff, true + default: + return "", false + } +} + +// UseTLS reports whether the broker connection should use TLS. +// +// brokerAdvertisedTLS is what mDNS discovery learned about the server (the +// "nats_tls" TXT record); it is false when the URL was configured explicitly, +// since nothing was discovered. It only participates in "auto". +func (c NATSConfig) UseTLS(brokerAdvertisedTLS bool) bool { + mode, _ := normalizeTLSMode(c.TLSEnabled) + switch mode { + case TLSOn: + return true + case TLSOff: + return false + default: // TLSAuto, including the "" zero value + return brokerAdvertisedTLS || + c.TLSCertFile != "" || c.TLSKeyFile != "" || + c.TLSCAFile != "" || c.InsecureSkipVerify + } } // WorkerSettings controls the worker's identity and runtime behavior. @@ -459,6 +540,28 @@ type MetricsConfig struct { // EnablePprof exposes Go runtime profiling endpoints at /debug/pprof/. // Env: SQI_WORKER_METRICS_ENABLE_PPROF EnablePprof bool `yaml:"enable_pprof"` + + // TLS terminates TLS on this listener. Off by default, which is right for + // the loopback default address; turn it on together with any Addr that is + // reachable from elsewhere, since this endpoint carries metrics, health and + // — if EnablePprof is set — full runtime profiles. + TLS MetricsTLSConfig `yaml:"tls"` +} + +// MetricsTLSConfig terminates TLS on the worker's metrics/health listener. +type MetricsTLSConfig struct { + // Enabled serves HTTPS on metrics.addr. There is no plaintext port when it + // is on: the listener upgrades in place, as the server's does. + // Env: SQI_WORKER_METRICS_TLS_ENABLED + Enabled bool `yaml:"enabled"` + + // CertFile is the PEM certificate served on this listener. + // Env: SQI_WORKER_METRICS_TLS_CERT_FILE + CertFile string `yaml:"cert_file"` + + // KeyFile is the PEM private key matching CertFile. + // Env: SQI_WORKER_METRICS_TLS_KEY_FILE + KeyFile string `yaml:"key_file"` } // DiscoveryConfig controls mDNS-based sqi-server auto-discovery. @@ -498,6 +601,7 @@ func Default() WorkerConfig { } return WorkerConfig{ NATS: NATSConfig{ + TLSEnabled: TLSAuto, MaxReconnectAttempts: -1, ReconnectWait: 2 * time.Second, }, @@ -757,9 +861,12 @@ func applyStagingEnv(c *StagingConfig) { } } -func applyNATSEnv(c *NATSConfig) { - if v := os.Getenv("SQI_WORKER_NATS_URL"); v != "" { - c.URL = v +// applyNATSTLSEnv overlays the NATS TLS environment variables onto c. Split +// out of [applyNATSEnv] to keep its cyclomatic complexity under the lint +// threshold. +func applyNATSTLSEnv(c *NATSConfig) { + if v := os.Getenv("SQI_WORKER_NATS_TLS_ENABLED"); v != "" { + c.TLSEnabled = v } if v := os.Getenv("SQI_WORKER_NATS_TLS_CERT_FILE"); v != "" { c.TLSCertFile = v @@ -773,6 +880,19 @@ func applyNATSEnv(c *NATSConfig) { if v := os.Getenv("SQI_WORKER_NATS_INSECURE_SKIP_VERIFY"); v != "" { c.InsecureSkipVerify = parseBoolEnv(v) } + if v := os.Getenv("SQI_WORKER_NATS_SERVER_TLS_CA_FILE"); v != "" { + c.ServerTLSCAFile = v + } + if v := os.Getenv("SQI_WORKER_NATS_SERVER_TLS_INSECURE_SKIP_VERIFY"); v != "" { + c.ServerTLSInsecureSkipVerify = parseBoolEnv(v) + } +} + +func applyNATSEnv(c *NATSConfig) { + if v := os.Getenv("SQI_WORKER_NATS_URL"); v != "" { + c.URL = v + } + applyNATSTLSEnv(c) if v := os.Getenv("SQI_WORKER_NATS_MAX_RECONNECT_ATTEMPTS"); v != "" { if n, err := strconv.Atoi(v); err == nil { c.MaxReconnectAttempts = n @@ -873,6 +993,15 @@ func applyMetricsEnv(c *MetricsConfig) { if v := os.Getenv("SQI_WORKER_METRICS_ADDR"); v != "" { c.Addr = v } + if v := os.Getenv("SQI_WORKER_METRICS_TLS_ENABLED"); v != "" { + c.TLS.Enabled = parseBoolEnv(v) + } + if v := os.Getenv("SQI_WORKER_METRICS_TLS_CERT_FILE"); v != "" { + c.TLS.CertFile = v + } + if v := os.Getenv("SQI_WORKER_METRICS_TLS_KEY_FILE"); v != "" { + c.TLS.KeyFile = v + } if v := os.Getenv("SQI_WORKER_METRICS_ENABLE_PPROF"); v != "" { c.EnablePprof = parseBoolEnv(v) } @@ -1017,10 +1146,63 @@ func Validate(cfg WorkerConfig) []ValidationError { errs = append(errs, validateIsolation(cfg)...) errs = append(errs, validateExpr(cfg.Expr)...) errs = append(errs, validateQueueIDs(cfg.Worker.QueueIDs)...) + errs = append(errs, validateNATSTLSMode(cfg.NATS)...) + errs = append(errs, validateMetricsTLS(cfg.Metrics)...) return errs } +// validateMetricsTLS refuses a metrics TLS block that cannot serve, at load +// rather than when the first scrape arrives. +func validateMetricsTLS(cfg MetricsConfig) []ValidationError { + if !cfg.TLS.Enabled { + return nil + } + var errs []ValidationError + if cfg.TLS.CertFile == "" { + errs = append(errs, ValidationError{ + Field: "metrics.tls.cert_file", + Message: "must be set when metrics.tls.enabled is true", + }) + } + if cfg.TLS.KeyFile == "" { + errs = append(errs, ValidationError{ + Field: "metrics.tls.key_file", + Message: "must be set when metrics.tls.enabled is true", + }) + } + if len(errs) > 0 { + return errs + } + if _, err := tls.LoadX509KeyPair(cfg.TLS.CertFile, cfg.TLS.KeyFile); err != nil { + return []ValidationError{{ + Field: "metrics.tls.cert_file", + Message: fmt.Sprintf("cannot load certificate/key pair (%s, %s): %s", cfg.TLS.CertFile, cfg.TLS.KeyFile, err), + }} + } + return nil +} + +// validateNATSTLSMode rejects a nats.tls_enabled value that is not one of the +// three modes or one of their accepted synonyms. An unrecognized value would +// otherwise fall into the "auto" default and silently mean something the +// operator did not write. ("yes" and "1" ARE recognized — see +// normalizeTLSMode.) +func validateNATSTLSMode(cfg NATSConfig) []ValidationError { + if _, ok := normalizeTLSMode(cfg.TLSEnabled); ok { + return nil + } + return []ValidationError{ + { + Field: "nats.tls_enabled", + Message: fmt.Sprintf( + "must be %q, %q or %q, got %q; %q infers TLS from the other tls_* settings and from mDNS discovery", + TLSAuto, TLSOn, TLSOff, cfg.TLSEnabled, TLSAuto, + ), + }, + } +} + // validateExpr rejects an out-of-range expr: value at STARTUP rather than // clamping it. A clamp would leave an operator who typed 100 believing they // had tightened the host when they had not. diff --git a/internal/worker/config/config_test.go b/internal/worker/config/config_test.go index 2f9cfef5..dca08ccd 100644 --- a/internal/worker/config/config_test.go +++ b/internal/worker/config/config_test.go @@ -746,3 +746,91 @@ func TestLoad_NATSCredentialFileExplicitValuePreserved(t *testing.T) { t.Errorf("NATS.CredentialFile = %q, want /etc/sqi/worker.nk", cfg.NATS.CredentialFile) } } + +func TestNATSConfig_UseTLS(t *testing.T) { + tests := []struct { + name string + cfg NATSConfig + advertisedTLS bool + want bool + }{ + {"zero value behaves as auto", NATSConfig{}, false, false}, + {"auto, nothing set", NATSConfig{TLSEnabled: TLSAuto}, false, false}, + {"auto infers from ca file", NATSConfig{TLSEnabled: TLSAuto, TLSCAFile: "/ca.crt"}, false, true}, + {"auto infers from client cert", NATSConfig{TLSEnabled: TLSAuto, TLSCertFile: "/c.crt"}, false, true}, + {"auto infers from insecure_skip_verify", NATSConfig{TLSEnabled: TLSAuto, InsecureSkipVerify: true}, false, true}, + + // The discovery signal is an inference input, and ONLY for auto. + {"auto infers from a TLS-advertising server", NATSConfig{TLSEnabled: TLSAuto}, true, true}, + {"zero value infers from a TLS-advertising server", NATSConfig{}, true, true}, + + {"true forces on with nothing configured", NATSConfig{TLSEnabled: TLSOn}, false, true}, + {"true ignores a non-advertising server", NATSConfig{TLSEnabled: TLSOn}, false, true}, + + // The state the old force-on-only boolean could not express. + {"false forces off despite a ca file", NATSConfig{TLSEnabled: TLSOff, TLSCAFile: "/ca.crt"}, false, false}, + {"false forces off despite discovery", NATSConfig{TLSEnabled: TLSOff}, true, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.cfg.UseTLS(tt.advertisedTLS); got != tt.want { + t.Errorf("UseTLS(%v) = %v, want %v", tt.advertisedTLS, got, tt.want) + } + }) + } +} + +func TestValidate_NATSTLSMode(t *testing.T) { + for _, mode := range []string{"", TLSAuto, TLSOn, TLSOff, "1", "0", "yes", "off"} { + cfg := Default() + cfg.NATS.URL = "nats://localhost:4222" + cfg.NATS.TLSEnabled = mode + for _, e := range Validate(cfg) { + if e.Field == "nats.tls_enabled" { + t.Errorf("mode %q was rejected: %s", mode, e.Message) + } + } + } + + cfg := Default() + cfg.NATS.URL = "nats://localhost:4222" + cfg.NATS.TLSEnabled = "maybe" + var found bool + for _, e := range Validate(cfg) { + if e.Field == "nats.tls_enabled" { + found = true + if !strings.Contains(e.Message, "auto") { + t.Errorf("message does not mention the auto mode: %s", e.Message) + } + } + } + if !found { + // An unrecognized value would otherwise fall into the auto default and + // silently mean something the operator did not write. ("yes" and "1" + // ARE recognized — see TestNormalizeTLSMode_AcceptsBoolSynonyms.) + t.Error("nats.tls_enabled = \"maybe\" was accepted") + } +} + +func TestNormalizeTLSMode_AcceptsBoolSynonyms(t *testing.T) { + // This key looked like a bool until recently, and SQI_WORKER_NATS_TLS_ENABLED=1 + // is how people set boolean-looking env vars. YAML and env must agree. + for _, v := range []string{"1", "yes", "on", "TRUE", " true "} { + if mode, ok := normalizeTLSMode(v); !ok || mode != TLSOn { + t.Errorf("normalizeTLSMode(%q) = (%q, %v), want (%q, true)", v, mode, ok, TLSOn) + } + } + for _, v := range []string{"0", "no", "off", "FALSE"} { + if mode, ok := normalizeTLSMode(v); !ok || mode != TLSOff { + t.Errorf("normalizeTLSMode(%q) = (%q, %v), want (%q, true)", v, mode, ok, TLSOff) + } + } + for _, v := range []string{"", "auto", "AUTO"} { + if mode, ok := normalizeTLSMode(v); !ok || mode != TLSAuto { + t.Errorf("normalizeTLSMode(%q) = (%q, %v), want (%q, true)", v, mode, ok, TLSAuto) + } + } + if _, ok := normalizeTLSMode("maybe"); ok { + t.Error("normalizeTLSMode accepted \"maybe\"") + } +} diff --git a/internal/worker/discovery/applytls.go b/internal/worker/discovery/applytls.go new file mode 100644 index 00000000..5c34a56a --- /dev/null +++ b/internal/worker/discovery/applytls.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package discovery + +import ( + "context" + "log/slog" + "strings" + + workerconfig "github.com/uberware/sqi/internal/worker/config" +) + +// ApplyTLS folds what mDNS learned about the server into the worker's NATS +// configuration. +// +// This is what makes nats.tls_enabled's "auto" mode able to see a TLS-required +// broker. Without it a worker that discovers its server has no way to know the +// broker needs TLS, attempts plaintext, and fails with nats.go's +// "secure connection not available" — which names neither the cause nor the +// setting that fixes it. +// +// An explicit nats.url discovers nothing, so found's TLS fields are false and +// this is a no-op: an operator who configures the URL by hand also configures +// the transport by hand. +// +// On trusting an advertisement at all: the signal cannot be used to weaken the +// connection. Forging nats_tls=1 UPGRADES a worker to TLS; withholding it +// leaves the worker attempting plaintext against a TLS broker, which fails +// closed. Any locally configured tls_* setting still wins, so an advertisement +// can never downgrade a worker that was configured for TLS. And an attacker +// able to forge mDNS already controls NATSURL entirely, so the TLS record adds +// no surface that was not already there. +func ApplyTLS(ctx context.Context, cfg *workerconfig.NATSConfig, found Result, logger *slog.Logger) { + switch { + case found.NATSTLS && cfg.TLSEnabled == workerconfig.TLSOff: + // The operator forced plaintext against a broker that requires TLS. + // Honor it — "false" means false — but say plainly why the connection + // is about to fail, since the broker's own error will not mention this + // setting. + logger.WarnContext(ctx, + "discovery: server advertises a TLS-required broker but nats.tls_enabled is \"false\"; "+ + "the connection will be refused — set it to \"auto\" or \"true\"", + slog.String("url", cfg.URL)) + + case found.NATSTLS && !cfg.UseTLS(false): + // "auto" with nothing configured locally. The advertisement is the + // reason to use TLS, so use it. With no CA configured this verifies + // against the system roots, which fails on a farm CA with a readable + // certificate error naming the issuer — a far better diagnostic than + // a plaintext attempt produces. + logger.InfoContext(ctx, + "discovery: server advertises a TLS-required broker; enabling TLS for this connection", + slog.String("url", cfg.URL)) + cfg.TLSEnabled = workerconfig.TLSOn + if cfg.TLSCAFile == "" && !cfg.InsecureSkipVerify { + logger.WarnContext(ctx, + "discovery: no nats.tls_ca_file is configured, so the broker certificate is verified "+ + "against the system roots; a farm CA will not verify — copy the server's ca.crt "+ + "and set nats.tls_ca_file") + } + } + + // Enrollment never uses a discovered URL, so this cannot be corrected + // automatically — but an http:// server_url against a server advertising + // HTTPS is a misconfiguration worth naming before the request fails. + if found.HTTPTLS && strings.HasPrefix(cfg.ServerURL, "http://") { + logger.WarnContext(ctx, + "discovery: server advertises HTTPS but nats.server_url is http://; "+ + "enrollment will fail — use https:// and set nats.server_tls_ca_file", + slog.String("server_url", cfg.ServerURL)) + } +} diff --git a/internal/worker/discovery/applytls_test.go b/internal/worker/discovery/applytls_test.go new file mode 100644 index 00000000..89c86b7d --- /dev/null +++ b/internal/worker/discovery/applytls_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package discovery + +import ( + "bytes" + "context" + "log/slog" + "strings" + "testing" + + workerconfig "github.com/uberware/sqi/internal/worker/config" +) + +// ApplyTLS is what makes nats.tls_enabled's "auto" mode able to see a +// TLS-required broker. Before it existed the server advertised tls=1 / +// nats_tls=1 over mDNS and nothing read them, so a discovering worker attempted +// plaintext and failed with nats.go's "secure connection not available" — +// naming neither the cause nor the setting that fixes it. + +func applyFixture(t *testing.T, cfg workerconfig.NATSConfig, found Result) (workerconfig.NATSConfig, string) { + t.Helper() + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})) + ApplyTLS(context.Background(), &cfg, found, logger) + return cfg, buf.String() +} + +func TestApplyTLS_AutoEnablesOnAdvertisedBroker(t *testing.T) { + got, logs := applyFixture( + t, + workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto}, + Result{NATSURL: "nats://srv:4222", NATSTLS: true}, + ) + if !got.UseTLS(false) { + t.Error("auto did not enable TLS against a broker advertising nats_tls=1") + } + if !strings.Contains(logs, "enabling TLS") { + t.Errorf("no log explained the switch:\n%s", logs) + } + // With no CA the system roots are used, which cannot verify a farm CA. + // Saying so up front beats an unexplained verification failure. + if !strings.Contains(logs, "nats.tls_ca_file") { + t.Errorf("no log warned about the missing CA:\n%s", logs) + } +} + +func TestApplyTLS_AutoWithCAStaysQuietAboutTheCA(t *testing.T) { + _, logs := applyFixture( + t, + workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto, TLSCAFile: "/etc/sqi/ca.crt"}, + Result{NATSURL: "nats://srv:4222", NATSTLS: true}, + ) + if strings.Contains(logs, "no nats.tls_ca_file is configured") { + t.Errorf("warned about a missing CA when one is configured:\n%s", logs) + } +} + +func TestApplyTLS_ForcedOffIsHonoredButExplained(t *testing.T) { + got, logs := applyFixture( + t, + workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSOff}, + Result{NATSURL: "nats://srv:4222", NATSTLS: true}, + ) + // "false" means false: discovery must not override an explicit decision. + if got.TLSEnabled != workerconfig.TLSOff || got.UseTLS(true) { + t.Error("discovery overrode an explicit nats.tls_enabled=false") + } + if !strings.Contains(logs, "will be refused") { + t.Errorf("no log warned that the connection is about to fail:\n%s", logs) + } +} + +func TestApplyTLS_PlaintextServerChangesNothing(t *testing.T) { + // The default path: a plaintext farm must be untouched. + got, logs := applyFixture( + t, + workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto}, + Result{NATSURL: "nats://srv:4222"}, + ) + if got.UseTLS(false) { + t.Error("TLS was enabled against a plaintext server") + } + if logs != "" { + t.Errorf("a plaintext discovery produced output:\n%s", logs) + } +} + +func TestApplyTLS_ExplicitURLIsANoOp(t *testing.T) { + // An explicit nats.url discovers nothing, so Result carries no TLS state + // and an operator who set the URL by hand keeps full control. + got, logs := applyFixture( + t, + workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto}, + Result{NATSURL: "nats://explicit:4222"}, + ) + if got.UseTLS(false) || logs != "" { + t.Errorf("explicit URL was not a no-op: cfg=%+v logs=%s", got, logs) + } +} + +func TestApplyTLS_WarnsOnHTTPServerURLAgainstHTTPSServer(t *testing.T) { + _, logs := applyFixture( + t, + workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto, ServerURL: "http://srv:8080"}, + Result{NATSURL: "nats://srv:4222", HTTPTLS: true}, + ) + if !strings.Contains(logs, "server_url") { + t.Errorf("no log warned about the http:// enrollment URL:\n%s", logs) + } + + // An https:// server_url against the same server is correct and silent. + _, quiet := applyFixture( + t, + workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto, ServerURL: "https://srv:8080"}, + Result{NATSURL: "nats://srv:4222", HTTPTLS: true}, + ) + if strings.Contains(quiet, "server_url") { + t.Errorf("warned about a correct https:// server_url:\n%s", quiet) + } +} diff --git a/internal/worker/discovery/browser.go b/internal/worker/discovery/browser.go index 6589796e..9e146cb5 100644 --- a/internal/worker/discovery/browser.go +++ b/internal/worker/discovery/browser.go @@ -40,6 +40,11 @@ const ( // domain is the standard mDNS multicast domain. domain = "local." + + // hostLookupTimeout bounds the check in resolveHost. It is short on + // purpose: a name that needs longer than this is not one a worker should + // wait on at every boot, and the advertised IP is right there. + hostLookupTimeout = 2 * time.Second ) // Result holds the information extracted from a discovered sqi-server @@ -56,6 +61,30 @@ type Result struct { // Version is the server build version from the "version" TXT record. Version string + + // NATSTLS reports whether the discovered server's broker requires TLS, + // from the "nats_tls" TXT record. A server that predates the record, or + // one serving a plaintext broker, leaves this false. + // + // This is an INPUT to nats.tls_enabled's "auto" mode: it is the one signal + // that can tell a worker the broker needs TLS without the operator having + // configured anything locally. + NATSTLS bool + + // ipHost is the IP address the advertisement carried, kept so a hostname + // that does not resolve on this host can still be reached. Unexported: + // callers want NATSURL, not the decision behind it. + ipHost string + + // natsPort is the advertised broker port, kept for the same reason. + natsPort int + + // HTTPTLS reports whether the discovered server's REST API is + // TLS-terminated, from the "tls" TXT record. Enrollment does not use a + // discovered URL (nats.server_url is always explicit), so this is used to + // warn about an http:// server_url pointed at an https:// server rather + // than to rewrite anything. + HTTPTLS bool } // ErrDiscoveryTimeout is returned by [Browse] when the timeout expires with @@ -140,6 +169,9 @@ func browseService(ctx context.Context, service string, timeout time.Duration, l ) continue } + // The advertised hostname is only useful if this host can resolve + // it; fall back to the advertised IP when it cannot. + result = resolveHost(ctx, result, logger) logger.InfoContext( ctx, "discovery: found sqi-server via mDNS", slog.String("instance", result.InstanceName), @@ -192,6 +224,11 @@ func entryToResult(entry *zeroconf.ServiceEntry) (Result, error) { // Strip trailing dot from mDNS hostname (e.g. "myhost.local." → "myhost.local") host = strings.TrimSuffix(host, ".") + // Keep the advertised IP even when a hostname is present. mDNS gives us + // both, and the hostname is only useful to a host that can resolve it — + // see resolveHost. + ipHost := advertisedIP(entry) + txt := parseTXTRecords(entry.Text) natsPortStr, ok := txt["nats"] @@ -205,12 +242,98 @@ func entryToResult(entry *zeroconf.ServiceEntry) (Result, error) { return Result{ NATSURL: "nats://" + net.JoinHostPort(host, strconv.Itoa(natsPort)), + ipHost: ipHost, + natsPort: natsPort, InstanceName: entry.Instance, InstanceID: txt["id"], Version: txt["version"], + NATSTLS: txt["nats_tls"] == "1", + HTTPTLS: txt["tls"] == "1", }, nil } +// advertisedIP returns a usable IP address from the mDNS entry, or "" when it +// carried none. +// +// "Usable" excludes link-local addresses. A responder publishes every address +// on the interfaces it advertised on, and on loopback that includes fe80::1 — +// which cannot be dialed without a zone index, so a worker that picked it would +// fail with "no route to host" having had a working address in the same entry. +// +// IPv4 is preferred simply because it is likelier to be reachable and to appear +// in a certificate's SANs. +func advertisedIP(entry *zeroconf.ServiceEntry) string { + for _, ip := range entry.AddrIPv4 { + if !usableIP(ip) { + continue + } + // To4() for the dotted-decimal form: String() on a v4-mapped IPv6 + // address returns "::ffff:a.b.c.d", which is not a valid URL host. + if ip4 := ip.To4(); ip4 != nil { + return ip4.String() + } + return ip.String() + } + for _, ip := range entry.AddrIPv6 { + if usableIP(ip) { + return ip.String() + } + } + return "" +} + +// usableIP reports whether ip can be dialed from this host as written. +func usableIP(ip net.IP) bool { + return ip != nil && + !ip.IsUnspecified() && + !ip.IsLinkLocalUnicast() && + !ip.IsLinkLocalMulticast() +} + +// lookupHost is net.DefaultResolver.LookupHost, replaced in tests. +var lookupHost = net.DefaultResolver.LookupHost + +// resolveHost returns r with NATSURL rewritten to the advertised IP when the +// advertised HOSTNAME cannot be resolved on this machine. +// +// mDNS advertises ".local", and resolving that needs an mDNS NSS +// module. macOS has one built in; a headless Linux worker generally does not +// (no avahi/nss-mdns), and there the worker would discover its server correctly +// and then fail to dial it — "no servers available", which names neither the +// cause nor the fix. The address was in the advertisement all along. +// +// The hostname is preferred when it does resolve, because a TLS broker's +// certificate names hosts far more often than IPs: `sqi-server tls init` puts +// the machine's hostname in the SANs, not its LAN address. Falling back to an +// IP against a TLS broker therefore trades an unresolvable name for a +// certificate mismatch — better, because that error says what is wrong, but +// worth warning about. +func resolveHost(ctx context.Context, r Result, logger *slog.Logger) Result { + if r.ipHost == "" || r.natsPort == 0 { + return r // nothing to fall back to + } + host, _, err := net.SplitHostPort(strings.TrimPrefix(r.NATSURL, "nats://")) + if err != nil || net.ParseIP(host) != nil { + return r // already an IP + } + + lookupCtx, cancel := context.WithTimeout(ctx, hostLookupTimeout) + defer cancel() + if addrs, err := lookupHost(lookupCtx, host); err == nil && len(addrs) > 0 { + return r + } + + r.NATSURL = "nats://" + net.JoinHostPort(r.ipHost, strconv.Itoa(r.natsPort)) + logger.WarnContext(ctx, + "discovery: the advertised hostname does not resolve here; using the advertised IP instead", + slog.String("hostname", host), + slog.String("url", r.NATSURL), + slog.String("note", "if the broker uses TLS, its certificate must cover this IP "+ + "(sqi-server tls init names the hostname, not the address) — install an mDNS "+ + "resolver such as libnss-mdns, or add a hosts entry, to use the name instead")) + return r +} + // parseTXTRecords converts a slice of "key=value" DNS TXT strings into a map. // Records without an "=" are stored with an empty string value. // First occurrence wins on duplicate keys so that a well-formed record is not @@ -227,35 +350,37 @@ func parseTXTRecords(records []string) map[string]string { return out } -// ResolveNATSURL returns the NATS URL to connect to, applying the following -// precedence: +// Resolve returns the discovered server, applying the following precedence: // // 1. If explicitURL is non-empty, return it directly (mDNS bypassed entirely). // 2. If mdnsEnabled is false, return an error with a clear message. -// 3. Run [Browse] with the given timeout and return the discovered URL. +// 3. Run [Browse] and return what it found. +// +// It returns the whole [Result], not just the URL: the TLS records travel with +// it, and a caller that only took the URL would silently drop the one signal +// that can tell a worker its broker needs TLS. +// +// An explicit URL yields a Result with only NATSURL set — nothing was +// discovered, so nothing is known about the server's transport, and the TLS +// fields stay false rather than claiming otherwise. // // This is the single entry point that start.go calls before dialing NATS. -func ResolveNATSURL(ctx context.Context, explicitURL string, mdnsEnabled bool, timeout time.Duration, logger *slog.Logger) (string, error) { +func Resolve(ctx context.Context, explicitURL string, mdnsEnabled bool, timeout time.Duration, logger *slog.Logger) (Result, error) { // Explicit URL bypasses mDNS entirely. if explicitURL != "" { logger.InfoContext( ctx, "discovery: using explicit NATS URL (mDNS bypassed)", slog.String("url", explicitURL), ) - return explicitURL, nil + return Result{NATSURL: explicitURL}, nil } // MDNS disabled — log clearly and return actionable error. if !mdnsEnabled { logger.ErrorContext(ctx, "discovery: mDNS is disabled and no explicit NATS URL is configured; "+ "set nats.url in configuration or enable discovery.enable_mdns") - return "", ErrDiscoveryDisabled + return Result{}, ErrDiscoveryDisabled } - // Browse the local network. - result, err := Browse(ctx, timeout, logger) - if err != nil { - return "", err - } - return result.NATSURL, nil + return Browse(ctx, timeout, logger) } diff --git a/internal/worker/discovery/browser_test.go b/internal/worker/discovery/browser_test.go index 8f2bbbcb..4fd896e2 100644 --- a/internal/worker/discovery/browser_test.go +++ b/internal/worker/discovery/browser_test.go @@ -231,9 +231,9 @@ func TestEntryToResultFields(t *testing.T) { } } -// ── ResolveNATSURL ──────────────────────────────────────────────────────────── +// ── Resolve ──────────────────────────────────────────────────────────── -func TestResolveNATSURLExplicitBypassesMDNS(t *testing.T) { +func TestResolveExplicitBypassesMDNS(t *testing.T) { t.Parallel() ctx := context.Background() @@ -244,7 +244,8 @@ func TestResolveNATSURLExplicitBypassesMDNS(t *testing.T) { for _, enabled := range []bool{true, false} { t.Run("mdns_enabled="+boolStr(enabled), func(t *testing.T) { t.Parallel() - url, err := ResolveNATSURL(ctx, "nats://explicit:4222", enabled, time.Second, logger) + got, err := Resolve(ctx, "nats://explicit:4222", enabled, time.Second, logger) + url := got.NATSURL if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -255,13 +256,13 @@ func TestResolveNATSURLExplicitBypassesMDNS(t *testing.T) { } } -func TestResolveNATSURLDisabledNoURL(t *testing.T) { +func TestResolveDisabledNoURL(t *testing.T) { t.Parallel() ctx := context.Background() logger := testLogger() - _, err := ResolveNATSURL(ctx, "", false, time.Second, logger) + _, err := Resolve(ctx, "", false, time.Second, logger) if err == nil { t.Fatal("expected ErrDiscoveryDisabled, got nil") } @@ -299,14 +300,14 @@ func TestBrowseMDNSTimeout(t *testing.T) { } } -func TestResolveNATSURLContextCancelled(t *testing.T) { +func TestResolveContextCancelled(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) cancel() // cancel immediately before any I/O logger := testLogger() - _, err := ResolveNATSURL(ctx, "", true, 5*time.Second, logger) + _, err := Resolve(ctx, "", true, 5*time.Second, logger) if !errors.Is(err, context.Canceled) { t.Fatalf("expected context.Canceled, got %v", err) } @@ -320,3 +321,196 @@ func boolStr(b bool) string { } return "false" } + +// ── TLS TXT records ─────────────────────────────────────────────────────────── + +func TestEntryToResult_TLSRecords(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + txt []string + wantNATSTLS bool + wantHTTPTLS bool + }{ + { + // A plaintext server omits the keys entirely rather than sending + // "tls=0", so absence is the normal case and must read as false. + name: "absent keys mean plaintext", + txt: []string{"id=a", "nats=4222", "version=0.4.0", "http=8080"}, + }, + { + name: "nats_tls only", + txt: []string{"id=a", "nats=4222", "nats_tls=1"}, + wantNATSTLS: true, + }, + { + name: "tls only", + txt: []string{"id=a", "nats=4222", "tls=1"}, + wantHTTPTLS: true, + }, + { + name: "both", + txt: []string{"id=a", "nats=4222", "tls=1", "nats_tls=1"}, + wantNATSTLS: true, + wantHTTPTLS: true, + }, + { + // Only "1" counts. Anything else is a server we do not understand, + // and guessing "probably TLS" would break a plaintext farm. + name: "unexpected values are not truthy", + txt: []string{"id=a", "nats=4222", "tls=true", "nats_tls=yes"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := entryToResult(buildZeroconfEntry("render01.local.", tt.txt, "srv")) + if err != nil { + t.Fatalf("entryToResult: %v", err) + } + if got.NATSTLS != tt.wantNATSTLS { + t.Errorf("NATSTLS = %v, want %v", got.NATSTLS, tt.wantNATSTLS) + } + if got.HTTPTLS != tt.wantHTTPTLS { + t.Errorf("HTTPTLS = %v, want %v", got.HTTPTLS, tt.wantHTTPTLS) + } + }) + } +} + +func TestResolve_ExplicitURLClaimsNothingAboutTLS(t *testing.T) { + t.Parallel() + + // An explicit URL discovers nothing, so the TLS fields must stay false + // rather than asserting a transport nobody looked up. + got, err := Resolve(t.Context(), "nats://explicit:4222", true, time.Second, testLogger()) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if got.NATSTLS || got.HTTPTLS { + t.Errorf("explicit URL reported TLS state %+v, want both false", got) + } +} + +// ── hostname fallback ───────────────────────────────────────────────────────── +// +// mDNS advertises ".local", which needs an mDNS NSS module to +// resolve. macOS has one; a headless Linux worker generally does not. Without +// this fallback such a worker discovers its server and then cannot dial it, +// reporting "no servers available" — while the address it needed was in the +// advertisement all along. + +func TestResolveHost_KeepsAResolvableHostname(t *testing.T) { + orig := lookupHost + t.Cleanup(func() { lookupHost = orig }) + lookupHost = func(context.Context, string) ([]string, error) { return []string{"192.0.2.9"}, nil } + + in := Result{NATSURL: "nats://render01.local:4222", ipHost: "192.0.2.9", natsPort: 4222} + got := resolveHost(t.Context(), in, testLogger()) + if got.NATSURL != in.NATSURL { + // The hostname is preferred because a TLS certificate names hosts far + // more often than addresses. + t.Errorf("NATSURL = %q, want the hostname form %q kept", got.NATSURL, in.NATSURL) + } +} + +func TestResolveHost_FallsBackToTheAdvertisedIP(t *testing.T) { + orig := lookupHost + t.Cleanup(func() { lookupHost = orig }) + lookupHost = func(context.Context, string) ([]string, error) { + return nil, errors.New("no such host") + } + + in := Result{NATSURL: "nats://render01.local:4222", ipHost: "192.0.2.9", natsPort: 4222} + got := resolveHost(t.Context(), in, testLogger()) + if got.NATSURL != "nats://192.0.2.9:4222" { + t.Errorf("NATSURL = %q, want the advertised IP", got.NATSURL) + } +} + +func TestResolveHost_LeavesTheURLAloneWithNothingToFallBackTo(t *testing.T) { + orig := lookupHost + t.Cleanup(func() { lookupHost = orig }) + lookupHost = func(context.Context, string) ([]string, error) { + return nil, errors.New("no such host") + } + + // An entry that carried no address: there is nothing better to offer, and + // the unresolvable name is more informative than an empty URL. + in := Result{NATSURL: "nats://render01.local:4222"} + if got := resolveHost(t.Context(), in, testLogger()); got.NATSURL != in.NATSURL { + t.Errorf("NATSURL = %q, want it unchanged", got.NATSURL) + } +} + +func TestResolveHost_DoesNotLookUpAnIP(t *testing.T) { + orig := lookupHost + t.Cleanup(func() { lookupHost = orig }) + called := false + lookupHost = func(context.Context, string) ([]string, error) { + called = true + return nil, errors.New("should not be called") + } + + // Already an IP: a lookup would be a pointless round trip at every boot. + in := Result{NATSURL: "nats://192.0.2.9:4222", ipHost: "192.0.2.9", natsPort: 4222} + got := resolveHost(t.Context(), in, testLogger()) + if called { + t.Error("resolveHost performed a DNS lookup for an address") + } + if got.NATSURL != in.NATSURL { + t.Errorf("NATSURL = %q, want it unchanged", got.NATSURL) + } +} + +func TestEntryToResult_KeepsTheAdvertisedIPAlongsideTheHostname(t *testing.T) { + t.Parallel() + entry := buildZeroconfEntry("render01.local.", []string{"id=a", "nats=4222"}, "srv") + entry.AddrIPv4 = []net.IP{net.ParseIP("192.0.2.9")} + + got, err := entryToResult(entry) + if err != nil { + t.Fatalf("entryToResult: %v", err) + } + if got.NATSURL != "nats://render01.local:4222" { + t.Errorf("NATSURL = %q, want the hostname form", got.NATSURL) + } + // The address must survive even though the hostname was usable, or the + // fallback has nothing to work with. + if got.ipHost != "192.0.2.9" || got.natsPort != 4222 { + t.Errorf("ipHost/natsPort = %q/%d, want 192.0.2.9/4222", got.ipHost, got.natsPort) + } +} + +func TestAdvertisedIP_SkipsLinkLocal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + v4 []net.IP + v6 []net.IP + want string + }{ + {"none", nil, nil, ""}, + {"ipv4 preferred", []net.IP{net.ParseIP("192.0.2.9")}, []net.IP{net.ParseIP("2001:db8::1")}, "192.0.2.9"}, + // A responder publishes every address on the interfaces it advertised + // on. On loopback that includes fe80::1, which cannot be dialed without + // a zone index — picking it fails with "no route to host" while a + // working address sat in the same entry. + {"link-local ipv6 skipped for loopback", nil, []net.IP{net.ParseIP("fe80::1"), net.ParseIP("::1")}, "::1"}, + {"link-local ipv4 skipped", []net.IP{net.ParseIP("169.254.1.1"), net.ParseIP("192.0.2.9")}, nil, "192.0.2.9"}, + {"only link-local yields nothing", []net.IP{net.ParseIP("169.254.1.1")}, []net.IP{net.ParseIP("fe80::1")}, ""}, + {"unspecified skipped", []net.IP{net.ParseIP("0.0.0.0"), net.ParseIP("192.0.2.9")}, nil, "192.0.2.9"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + entry := buildZeroconfEntry("h.local.", []string{"nats=4222"}, "srv") + entry.AddrIPv4, entry.AddrIPv6 = tt.v4, tt.v6 + if got := advertisedIP(entry); got != tt.want { + t.Errorf("advertisedIP() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/worker/enroll/enroll.go b/internal/worker/enroll/enroll.go index 132de742..59d3464c 100644 --- a/internal/worker/enroll/enroll.go +++ b/internal/worker/enroll/enroll.go @@ -14,6 +14,7 @@ package enroll import ( "bytes" "context" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -22,8 +23,10 @@ import ( "net/http" "os" "strings" + "time" "github.com/uberware/sqi/internal/brokerauth" + "github.com/uberware/sqi/internal/tlsutil" ) // ErrNoCredential is returned when no credential file exists and no join @@ -54,9 +57,44 @@ type Config struct { // ServerURL is the sqi-server HTTP base URL used for enrollment. ServerURL string - // HTTPClient performs the enrollment request. When nil, - // http.DefaultClient is used. + // HTTPClient performs the enrollment request. When nil, one is built from + // TLSCAFile and InsecureSkipVerify below. HTTPClient *http.Client + + // TLSCAFile is the CA that verifies the server's certificate at + // ServerURL. Empty means the system roots; set means THAT CA only, so a + // farm CA pins the acceptable issuer rather than widening it. + TLSCAFile string + + // InsecureSkipVerify disables verification of the server's certificate. + // Development only. + InsecureSkipVerify bool +} + +// httpClient returns the client used for the enrollment request. +// +// It exists because http.DefaultClient cannot trust a private farm CA, and +// enrollment is the one REST call a worker makes BEFORE it has any credential +// at all — so without this a farm with an HTTPS server could not bootstrap. +func httpClient(cfg Config) (*http.Client, error) { + if cfg.HTTPClient != nil { + return cfg.HTTPClient, nil + } + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: cfg.InsecureSkipVerify, //nolint:gosec // G402: explicitly requested via nats.server_tls_insecure_skip_verify + } + if cfg.TLSCAFile != "" { + pool, err := tlsutil.CertPool(cfg.TLSCAFile) + if err != nil { + return nil, fmt.Errorf("worker: enrollment %w", err) + } + tlsCfg.RootCAs = pool + } + return &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{TLSClientConfig: tlsCfg}, + }, nil } // enrollRequest is the body of POST /api/v1/workers/enroll. @@ -167,9 +205,15 @@ func enrollWithServer(ctx context.Context, cfg Config, token, publicKey string) } req.Header.Set("Content-Type", "application/json") - client := cfg.HTTPClient - if client == nil { - client = http.DefaultClient + client, err := httpClient(cfg) + if err != nil { + return err + } + if cfg.HTTPClient == nil { + // This client is ours and is used exactly once. Without this the + // worker keeps an idle TLS connection to the server open for the rest + // of its life, for a request it will never repeat. + defer client.CloseIdleConnections() } resp, err := client.Do(req) diff --git a/internal/worker/enroll/tls_test.go b/internal/worker/enroll/tls_test.go new file mode 100644 index 00000000..25006f90 --- /dev/null +++ b/internal/worker/enroll/tls_test.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package enroll_test + +import ( + "context" + "crypto/tls" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" + "github.com/uberware/sqi/internal/worker/enroll" +) + +// enrollCerts generates a farm CA and a server keypair covering loopback, +// writes them into a temp dir, and returns (dir, tls.Certificate). +func enrollCerts(t *testing.T) (dir string, pair tls.Certificate) { + t.Helper() + dir = t.TempDir() + ca, err := certgen.NewCA("test farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"localhost", "127.0.0.1", "::1"}, 365*24*time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + pair, err = tls.X509KeyPair(leaf.CertPEM, leaf.KeyPEM) + if err != nil { + t.Fatalf("X509KeyPair: %v", err) + } + return dir, pair +} + +// enrollServer starts an HTTPS server presenting pair that answers the +// enrollment endpoint with 201. +func enrollServer(t *testing.T, pair tls.Certificate) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/workers/enroll", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) //nolint:errcheck // draining a request body in a test fixture + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{"status": "enrolled"}) //nolint:errcheck // test fixture response + }) + srv := httptest.NewUnstartedServer(mux) + srv.TLS = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}} + srv.StartTLS() + t.Cleanup(srv.Close) + return srv +} + +func testLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } + +func TestEnsureCredential_HTTPSWithFarmCA(t *testing.T) { + dir, pair := enrollCerts(t) + srv := enrollServer(t, pair) + credFile := filepath.Join(t.TempDir(), "worker.nk") + + seed, pub, err := enroll.EnsureCredential(context.Background(), enroll.Config{ + WorkerID: "worker-01", + CredentialFile: credFile, + JoinToken: "tok", + ServerURL: srv.URL, + TLSCAFile: filepath.Join(dir, "ca.crt"), + }, testLogger()) + if err != nil { + t.Fatalf("EnsureCredential over HTTPS with the farm CA: %v", err) + } + if len(seed) == 0 || pub == "" { + t.Fatal("enrollment returned an empty credential") + } + if _, err := os.Stat(credFile); err != nil { + t.Errorf("credential file was not written: %v", err) + } +} + +func TestEnsureCredential_HTTPSWrongCAFails(t *testing.T) { + _, pair := enrollCerts(t) + otherDir, _ := enrollCerts(t) // a different CA + srv := enrollServer(t, pair) + credFile := filepath.Join(t.TempDir(), "worker.nk") + + _, _, err := enroll.EnsureCredential(context.Background(), enroll.Config{ + WorkerID: "worker-01", + CredentialFile: credFile, + JoinToken: "tok", + ServerURL: srv.URL, + TLSCAFile: filepath.Join(otherDir, "ca.crt"), + }, testLogger()) + if err == nil { + t.Fatal("enrollment succeeded against a server signed by an untrusted CA") + } + if !strings.Contains(err.Error(), "certificate") { + t.Errorf("error = %v, want a certificate verification failure", err) + } + // A failed enrollment must never leave a seed behind that a later boot + // would silently reuse. + if _, statErr := os.Stat(credFile); statErr == nil { + t.Error("a credential file was written despite the enrollment failing") + } +} + +func TestEnsureCredential_HTTPSInsecureSkipVerify(t *testing.T) { + _, pair := enrollCerts(t) + otherDir, _ := enrollCerts(t) + srv := enrollServer(t, pair) + credFile := filepath.Join(t.TempDir(), "worker.nk") + + _, _, err := enroll.EnsureCredential(context.Background(), enroll.Config{ + WorkerID: "worker-01", + CredentialFile: credFile, + JoinToken: "tok", + ServerURL: srv.URL, + TLSCAFile: filepath.Join(otherDir, "ca.crt"), + InsecureSkipVerify: true, + }, testLogger()) + if err != nil { + t.Fatalf("EnsureCredential with InsecureSkipVerify: %v", err) + } +} + +func TestEnsureCredential_BadCAFileIsReported(t *testing.T) { + _, pair := enrollCerts(t) + srv := enrollServer(t, pair) + garbage := filepath.Join(t.TempDir(), "garbage.pem") + if err := os.WriteFile(garbage, []byte("not a certificate\n"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + _, _, err := enroll.EnsureCredential(context.Background(), enroll.Config{ + WorkerID: "worker-01", + CredentialFile: filepath.Join(t.TempDir(), "worker.nk"), + JoinToken: "tok", + ServerURL: srv.URL, + TLSCAFile: garbage, + }, testLogger()) + if err == nil { + t.Fatal("an unparseable CA file was accepted") + } + if !strings.Contains(err.Error(), "CA file") { + t.Errorf("error = %v, want it to name the CA file", err) + } +} diff --git a/internal/worker/natsclient/natsclient.go b/internal/worker/natsclient/natsclient.go index 24f284c6..4b3d10ae 100644 --- a/internal/worker/natsclient/natsclient.go +++ b/internal/worker/natsclient/natsclient.go @@ -25,18 +25,17 @@ package natsclient import ( "context" "crypto/tls" - "crypto/x509" "errors" "fmt" "log/slog" "math" "math/rand/v2" - "os" "time" nats "github.com/nats-io/nats.go" "github.com/uberware/sqi/internal/brokerauth" + "github.com/uberware/sqi/internal/tlsutil" workerconfig "github.com/uberware/sqi/internal/worker/config" ) @@ -304,8 +303,11 @@ func exponentialBackoff(base time.Duration) func(int) time.Duration { // - cfg.TLSCAFile — custom CA for server cert verification // - cfg.InsecureSkipVerify — disable server cert verification func buildTLSOptions(cfg workerconfig.NATSConfig) ([]nats.Option, error) { - wantTLS := cfg.TLSCertFile != "" || cfg.TLSKeyFile != "" || - cfg.TLSCAFile != "" || cfg.InsecureSkipVerify + // The three-valued nats.tls_enabled decides this; see NATSConfig.UseTLS. + // Any mDNS signal has already been folded into cfg by the caller (see + // discovery.ApplyTLS), so false here means "nothing, including mDNS, gave a + // reason to use TLS". + wantTLS := cfg.UseTLS(false) if !wantTLS { return nil, nil @@ -330,13 +332,9 @@ func buildTLSOptions(cfg workerconfig.NATSConfig) ([]nats.Option, error) { // Custom CA for verifying the server's certificate. if cfg.TLSCAFile != "" { - pem, err := os.ReadFile(cfg.TLSCAFile) + pool, err := tlsutil.CertPool(cfg.TLSCAFile) if err != nil { - return nil, fmt.Errorf("natsclient: read CA file: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(pem) { - return nil, fmt.Errorf("natsclient: no valid certificates found in CA file %s", cfg.TLSCAFile) + return nil, fmt.Errorf("natsclient: %w", err) } tlsCfg.RootCAs = pool } diff --git a/internal/worker/natsclient/natsclient_test.go b/internal/worker/natsclient/natsclient_test.go index c2bcfdc6..1e834904 100644 --- a/internal/worker/natsclient/natsclient_test.go +++ b/internal/worker/natsclient/natsclient_test.go @@ -57,6 +57,16 @@ func TestBuildTLSOptions(t *testing.T) { wantErr bool }{ {"no tls", workerconfig.NATSConfig{}, false, false}, + // "auto" is the default and infers from the other tls_* settings. + {"auto with a ca file infers tls", workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto, InsecureSkipVerify: true}, true, false}, + {"auto with nothing set stays plaintext", workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto}, false, false}, + // "true" forces TLS for a broker with a publicly-trusted certificate, + // where no CA file would signal intent. + {"true forces tls with system roots", workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSOn}, true, false}, + // "false" forces plaintext even against configured TLS material — + // the state a force-on-only boolean could not express. + {"false overrides a configured ca file", workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSOff, TLSCAFile: "/does/not/exist.pem"}, false, false}, + {"false overrides insecure_skip_verify", workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSOff, InsecureSkipVerify: true}, false, false}, {"insecure skip verify", workerconfig.NATSConfig{InsecureSkipVerify: true}, true, false}, {"cert without key errors", workerconfig.NATSConfig{TLSCertFile: "/x.crt"}, false, true}, {"bad ca file errors", workerconfig.NATSConfig{TLSCAFile: "/does/not/exist.pem"}, false, true}, diff --git a/internal/worker/obs/obs.go b/internal/worker/obs/obs.go index cf13abde..98359e94 100644 --- a/internal/worker/obs/obs.go +++ b/internal/worker/obs/obs.go @@ -24,6 +24,7 @@ package obs import ( "context" + "errors" "log/slog" "net/http" httppprof "net/http/pprof" @@ -31,6 +32,7 @@ import ( "github.com/uberware/sqi/internal/health" "github.com/uberware/sqi/internal/middleware" + "github.com/uberware/sqi/internal/tlsutil" workmetrics "github.com/uberware/sqi/internal/worker/metrics" ) @@ -43,6 +45,19 @@ const ( uptimeTick = 15 * time.Second ) +// TLSConfig terminates TLS on the worker's observability listener. +// +// This listener serves metrics, health and optionally pprof. It defaults to +// loopback, where plaintext is fine — but an operator scraping it from +// Prometheus has to bind it wider, and then everything on it, pprof included, +// crosses the network in the clear. Zero value leaves it plaintext, which is +// the default and unchanged. +type TLSConfig struct { + Enabled bool + CertFile string + KeyFile string +} + // Server is the local observability HTTP server for sqi-worker. // Create with [New]; start with [Run]; stop with [Shutdown]. type Server struct { @@ -51,6 +66,7 @@ type Server struct { logger *slog.Logger metrics *workmetrics.Metrics health *health.Registry + tlsCfg TLSConfig httpServer *http.Server } @@ -61,12 +77,14 @@ type Server struct { // - logger — used for request logging and startup/shutdown messages // - m — worker Prometheus metrics; /metrics serves this registry // - h — health registry consulted by /readyz +// - tlsCfg — optional TLS termination; zero value serves plaintext func New( addr string, enablePprof bool, logger *slog.Logger, m *workmetrics.Metrics, h *health.Registry, + tlsCfg TLSConfig, ) *Server { return &Server{ addr: addr, @@ -74,14 +92,16 @@ func New( logger: logger, metrics: m, health: h, + tlsCfg: tlsCfg, } } -// Run starts the HTTP server and blocks until ctx is canceled, at which point -// it performs a graceful shutdown and returns. +// Run starts the HTTP server and blocks until the listener stops — which +// happens when [Server.Shutdown] is called, NOT when ctx is canceled. Canceling +// ctx alone stops the uptime goroutine and leaves the listener serving. // // Run also starts a background goroutine that refreshes the uptime metric on -// [uptimeTick] intervals; that goroutine exits when ctx is canceled. +// [uptimeTick] intervals; that goroutine does exit when ctx is canceled. func (s *Server) Run(ctx context.Context) { mux := s.buildMux(ctx) @@ -116,7 +136,21 @@ func (s *Server) Run(ctx context.Context) { slog.Bool("pprof", s.enablePprof), ) - if err := s.httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // Certificates are loaded once, here, and handed to the server rather than + // passed as paths — the same reason internal/server does it: re-reading at + // listen time could pick up a half-written file mid-rotation. + serve := s.httpServer.ListenAndServe + if s.tlsCfg.Enabled { + tlsConf, err := tlsutil.ServerConfig(s.tlsCfg.CertFile, s.tlsCfg.KeyFile, "") + if err != nil { + s.logger.ErrorContext(ctx, "obs: tls", slog.Any("error", err)) + return + } + s.httpServer.TLSConfig = tlsConf + serve = func() error { return s.httpServer.ListenAndServeTLS("", "") } + } + + if err := serve(); err != nil && !errors.Is(err, http.ErrServerClosed) { s.logger.ErrorContext(ctx, "obs: http server error", slog.Any("error", err)) } } diff --git a/internal/worker/obs/obs_test.go b/internal/worker/obs/obs_test.go index 97a0cdbe..21661453 100644 --- a/internal/worker/obs/obs_test.go +++ b/internal/worker/obs/obs_test.go @@ -23,7 +23,7 @@ func newTestServer(t *testing.T, pprof bool, ready bool) *Server { })) } logger := slog.New(slog.DiscardHandler) - return New("127.0.0.1:0", pprof, logger, workmetrics.New(), reg) + return New("127.0.0.1:0", pprof, logger, workmetrics.New(), reg, TLSConfig{}) } func TestBuildMux_HealthzAlwaysOK(t *testing.T) { diff --git a/internal/worker/obs/tls_test.go b/internal/worker/obs/tls_test.go new file mode 100644 index 00000000..dfee7a2d --- /dev/null +++ b/internal/worker/obs/tls_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package obs_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" + "github.com/uberware/sqi/internal/health" + workmetrics "github.com/uberware/sqi/internal/worker/metrics" + "github.com/uberware/sqi/internal/worker/obs" +) + +// The worker's observability listener serves metrics, health and optionally +// pprof. It defaults to loopback, but an operator scraping it from Prometheus +// has to bind it wider — and until this, there was no way to protect it. It was +// the one listener sqi opens that TLS did not reach, which made the top line of +// docs/tls.md false. + +func obsCerts(t *testing.T) (certFile, keyFile, caFile string) { + t.Helper() + dir := t.TempDir() + ca, err := certgen.NewCA("worker obs CA", time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + leaf, err := ca.NewServerCert([]string{"localhost", "127.0.0.1", "::1"}, time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "obs", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + return filepath.Join(dir, "obs.crt"), filepath.Join(dir, "obs.key"), filepath.Join(dir, "ca.crt") +} + +func freeAddr(t *testing.T) string { + t.Helper() + var lc net.ListenConfig + l, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = l.Close() }() + return l.Addr().String() +} + +// runObs starts a server and returns its address once it accepts connections. +func runObs(t *testing.T, tlsCfg obs.TLSConfig) string { + t.Helper() + addr := freeAddr(t) + srv := obs.New(addr, false, slog.New(slog.DiscardHandler), + workmetrics.New(), health.NewRegistry(), tlsCfg) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); srv.Run(ctx) }() + t.Cleanup(func() { + // Run blocks on the listener and does not watch ctx; Shutdown is what + // stops it. Canceling alone would hang here. + shutCtx, shutCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutCancel() + srv.Shutdown(shutCtx) + cancel() + select { + case <-done: + case <-time.After(15 * time.Second): + t.Error("obs server did not stop") + } + }) + + dialer := &net.Dialer{Timeout: 200 * time.Millisecond} + deadline := time.Now().Add(10 * time.Second) + for !time.Now().After(deadline) { + if c, err := dialer.DialContext(context.Background(), "tcp", addr); err == nil { + _ = c.Close() + return addr + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("obs listener did not come up on %s", addr) + return "" +} + +func get(t *testing.T, client *http.Client, url string) (*http.Response, error) { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + t.Fatalf("build request: %v", err) + } + return client.Do(req) +} + +func TestObs_PlaintextByDefault(t *testing.T) { + // The default path must be byte-for-byte what it always was. + addr := runObs(t, obs.TLSConfig{}) + resp, err := get(t, &http.Client{Timeout: 5 * time.Second}, "http://"+addr+"/healthz") + if err != nil { + t.Fatalf("GET /healthz over plain HTTP: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } +} + +func TestObs_ServesHTTPSWhenEnabled(t *testing.T) { + certFile, keyFile, caFile := obsCerts(t) + addr := runObs(t, obs.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile}) + + pem, err := os.ReadFile(caFile) + if err != nil { + t.Fatalf("read ca: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("ca did not append") + } + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}, + } + + for _, path := range []string{"/healthz", "/metrics"} { + resp, err := get(t, client, "https://"+addr+path) + if err != nil { + t.Fatalf("GET %s over TLS: %v", path, err) + } + code := resp.StatusCode + _ = resp.Body.Close() + if code != http.StatusOK { + t.Errorf("%s status = %d, want 200", path, code) + } + } + + // And plaintext must no longer be served on that port. + if r, err := get(t, &http.Client{Timeout: 5 * time.Second}, "http://"+addr+"/healthz"); err == nil { + code := r.StatusCode + _ = r.Body.Close() + if code == http.StatusOK { + t.Error("plaintext request succeeded against a TLS-enabled obs listener") + } + } +} diff --git a/scripts/smoke.sh b/scripts/smoke.sh index d61de17f..3c945d4f 100644 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -14,16 +14,19 @@ # the VALUE its expressions resolved to — see the "EXPR job" section below for # why that value can only have been produced by the worker at phase 3. # -# The whole flow above runs TWICE, back to back: once with broker +# The whole flow above runs THREE TIMES, back to back: once with broker # authentication left off (the default an operator gets with no nats.auth -# configuration at all — this run is untouched by the second), and once with -# it turned on and the worker enrolling itself via a join token before it can -# connect. The auth-off run always goes first, so a failure immediately says -# which mode broke: a failure before the "MODE 2/2" banner is the default -# path regressing, which is the more serious of the two by a wide margin. +# configuration at all — the other two runs never touch it), once with it +# turned on and the worker enrolling itself via a join token before it can +# connect, and once with TLS on end to end — HTTPS for REST, wss:// for the +# WebSocket gateway, and a TLS-required broker, all against certificates +# generated by `sqi-server tls init`. The plain run always goes first, so a +# failure immediately says which mode broke: a failure before the "MODE 2/3" +# banner is the default path regressing, which is the most serious of the +# three by a wide margin. # # Usage: -# bash scripts/smoke.sh # builds binaries if missing, runs both modes +# bash scripts/smoke.sh # builds binaries if missing, runs all three modes # make smoke # same, via the Makefile # # Environment overrides: @@ -31,7 +34,7 @@ # SQI_WORKER_BIN path to a prebuilt sqi-worker (default: /bin/sqi-worker) # SQI_SMOKE_PYTHON python interpreter for the WS check (auto-detected otherwise) # -# Exit status: 0 only if every assertion passed in both modes; non-zero with a +# Exit status: 0 only if every assertion passed in all three modes; non-zero with a # clear message (and the relevant server/worker log tail) otherwise. set -euo pipefail @@ -169,7 +172,8 @@ fi # run_smoke_flow MODE boots a fresh server+worker pair and drives the whole # assertion set described at the top of this file against it. MODE is # "noauth" (broker authentication left off — every env block below behaves -# exactly as this script always has) or "brokerauth" (nats.auth.enabled=true, +# exactly as this script always has), "tls" (TLS on every listener, against +# `sqi-server tls init` output) or "brokerauth" (nats.auth.enabled=true, # with a join token minted before the server starts and handed to the worker # instead of nothing). # @@ -227,6 +231,22 @@ HTTP_ADDR="127.0.0.1:${HTTP_PORT}" NATS_ADDR="127.0.0.1:${NATS_PORT}" BASE_URL="http://${HTTP_ADDR}" +# In TLS mode, generate the farm CA and server certificate with the shipped +# generator, then point every client at it. CURL_CA is spliced unquoted into +# each curl invocation; it is empty in the other two modes, so their command +# lines are unchanged. +CURL_CA="" +CERT_DIR="" +if [ "$mode" = "tls" ]; then + CERT_DIR="${TMP_DIR}/certs" + log "generating TLS material with sqi-server tls init" + "$SERVER_BIN" tls init --out "$CERT_DIR" --host 127.0.0.1 >/dev/null \ + || fail "sqi-server tls init failed" + [ -f "${CERT_DIR}/server.crt" ] || fail "tls init did not write server.crt" + BASE_URL="https://${HTTP_ADDR}" + CURL_CA="--cacert ${CERT_DIR}/ca.crt" +fi + # In broker-auth mode, mint a join token BEFORE the server starts — the CLI # operates directly on the SQLite file (no running server required), exactly # the offline path an operator would use. The worker gets the raw token @@ -249,26 +269,30 @@ if [ "$mode" = "brokerauth" ]; then fi log "starting sqi-server (http=${HTTP_ADDR}, nats=${NATS_ADDR}, mode=${mode})" -if [ "$mode" = "brokerauth" ]; then - SQI_HTTP_ADDR="$HTTP_ADDR" \ - SQI_NATS_ADDR="$NATS_ADDR" \ - SQI_NATS_DATA_DIR="${TMP_DIR}/nats" \ - SQI_STORE_SQLITE_PATH="${TMP_DIR}/sqi.db" \ - SQI_DISCOVERY_ENABLED="false" \ - SQI_SCHEDULER_TICK_INTERVAL="100ms" \ - SQI_LOG_LEVEL="warn" \ - SQI_NATS_AUTH_ENABLED="true" \ - "$SERVER_BIN" serve >"$SERVER_LOG" 2>&1 & -else - SQI_HTTP_ADDR="$HTTP_ADDR" \ - SQI_NATS_ADDR="$NATS_ADDR" \ - SQI_NATS_DATA_DIR="${TMP_DIR}/nats" \ - SQI_STORE_SQLITE_PATH="${TMP_DIR}/sqi.db" \ - SQI_DISCOVERY_ENABLED="false" \ - SQI_SCHEDULER_TICK_INTERVAL="100ms" \ - SQI_LOG_LEVEL="warn" \ - "$SERVER_BIN" serve >"$SERVER_LOG" 2>&1 & -fi +# run_smoke_flow runs in a subshell, so these exports are scoped to this run. +# One common block plus per-mode extras: three near-identical copies is how a +# new default silently ends up applying to only two of the three modes. +export SQI_HTTP_ADDR="$HTTP_ADDR" +export SQI_NATS_ADDR="$NATS_ADDR" +export SQI_NATS_DATA_DIR="${TMP_DIR}/nats" +export SQI_STORE_SQLITE_PATH="${TMP_DIR}/sqi.db" +export SQI_DISCOVERY_ENABLED="false" +export SQI_SCHEDULER_TICK_INTERVAL="100ms" +export SQI_LOG_LEVEL="warn" +case "$mode" in + brokerauth) + export SQI_NATS_AUTH_ENABLED="true" + ;; + tls) + export SQI_HTTP_TLS_ENABLED="true" + export SQI_HTTP_TLS_CERT_FILE="${CERT_DIR}/server.crt" + export SQI_HTTP_TLS_KEY_FILE="${CERT_DIR}/server.key" + export SQI_NATS_TLS_ENABLED="true" + export SQI_NATS_TLS_CERT_FILE="${CERT_DIR}/server.crt" + export SQI_NATS_TLS_KEY_FILE="${CERT_DIR}/server.key" + ;; +esac +"$SERVER_BIN" serve >"$SERVER_LOG" 2>&1 & SERVER_PID=$! # Poll /readyz until 200 (bounded). Fail fast if the process exits early. @@ -278,7 +302,7 @@ for _ in $(seq 1 150); do log_tail "server log" "$SERVER_LOG" fail "sqi-server exited before becoming ready" fi - code="$(curl -s -o /dev/null -w '%{http_code}' "${BASE_URL}/readyz" 2>/dev/null || true)" + code="$(curl -s ${CURL_CA} -o /dev/null -w '%{http_code}' "${BASE_URL}/readyz" 2>/dev/null || true)" if [ "$code" = "200" ]; then ready=1 break @@ -290,14 +314,14 @@ log "server ready" # ── Create a farm and a queue ───────────────────────────────────────────────── -farm_json="$(curl -s -X POST "${BASE_URL}/api/v1/farms" \ +farm_json="$(curl -s ${CURL_CA} -X POST "${BASE_URL}/api/v1/farms" \ -H 'Content-Type: application/json' \ -d '{"name":"smoke farm"}')" FARM_ID="$(json_get "$farm_json" id)" [ -n "$FARM_ID" ] || fail "could not create farm (response: ${farm_json})" log "created farm ${FARM_ID}" -queue_json="$(curl -s -X POST "${BASE_URL}/api/v1/queues" \ +queue_json="$(curl -s ${CURL_CA} -X POST "${BASE_URL}/api/v1/queues" \ -H 'Content-Type: application/json' \ -d "{\"farm_id\":\"${FARM_ID}\",\"name\":\"smoke queue\"}")" QUEUE_ID="$(json_get "$queue_json" id)" @@ -307,38 +331,35 @@ log "created queue ${QUEUE_ID}" # ── Start the worker ────────────────────────────────────────────────────────── log "starting sqi-worker (nats=nats://${NATS_ADDR}, farm=${FARM_ID}, queue=${QUEUE_ID}, mode=${mode})" -if [ "$mode" = "brokerauth" ]; then - # No SQI_WORKER_NATS_CREDENTIAL_FILE: it defaults under - # SQI_WORKER_DATA_DIR, which is what makes this worker's enrolled - # credential land in its own fresh, per-run data directory. - SQI_WORKER_NATS_URL="nats://${NATS_ADDR}" \ - SQI_WORKER_DISCOVERY_ENABLE_MDNS="false" \ - SQI_WORKER_FARM_ID="$FARM_ID" \ - SQI_WORKER_QUEUE_IDS="$QUEUE_ID" \ - SQI_WORKER_DATA_DIR="${TMP_DIR}/worker-data" \ - SQI_WORKER_ALLOW_ROOT="true" \ - SQI_WORKER_LOG_LEVEL="warn" \ - SQI_WORKER_LOG_FORMAT="text" \ - SQI_WORKER_HEARTBEAT_INTERVAL="1s" \ - SQI_WORKER_PULL_IDLE_BACKOFF="300ms" \ - SQI_WORKER_METRICS_ADDR="127.0.0.1:$(free_port)" \ - SQI_WORKER_NATS_JOIN_TOKEN_FILE="$JOIN_TOKEN_FILE" \ - SQI_WORKER_NATS_SERVER_URL="$BASE_URL" \ - "$WORKER_BIN" start >"$WORKER_LOG" 2>&1 & -else - SQI_WORKER_NATS_URL="nats://${NATS_ADDR}" \ - SQI_WORKER_DISCOVERY_ENABLE_MDNS="false" \ - SQI_WORKER_FARM_ID="$FARM_ID" \ - SQI_WORKER_QUEUE_IDS="$QUEUE_ID" \ - SQI_WORKER_DATA_DIR="${TMP_DIR}/worker-data" \ - SQI_WORKER_ALLOW_ROOT="true" \ - SQI_WORKER_LOG_LEVEL="warn" \ - SQI_WORKER_LOG_FORMAT="text" \ - SQI_WORKER_HEARTBEAT_INTERVAL="1s" \ - SQI_WORKER_PULL_IDLE_BACKOFF="300ms" \ - SQI_WORKER_METRICS_ADDR="127.0.0.1:$(free_port)" \ - "$WORKER_BIN" start >"$WORKER_LOG" 2>&1 & -fi +# Same shape as the server block above: one common set, per-mode extras. +export SQI_WORKER_NATS_URL="nats://${NATS_ADDR}" +export SQI_WORKER_DISCOVERY_ENABLE_MDNS="false" +export SQI_WORKER_FARM_ID="$FARM_ID" +export SQI_WORKER_QUEUE_IDS="$QUEUE_ID" +export SQI_WORKER_DATA_DIR="${TMP_DIR}/worker-data" +export SQI_WORKER_ALLOW_ROOT="true" +export SQI_WORKER_LOG_LEVEL="warn" +export SQI_WORKER_LOG_FORMAT="text" +export SQI_WORKER_HEARTBEAT_INTERVAL="1s" +export SQI_WORKER_PULL_IDLE_BACKOFF="300ms" +# Declared and assigned separately so the command substitution's exit status +# is not masked by `export` (shellcheck SC2155). +WORKER_METRICS_PORT="$(free_port)" +export SQI_WORKER_METRICS_ADDR="127.0.0.1:${WORKER_METRICS_PORT}" +case "$mode" in + brokerauth) + # No SQI_WORKER_NATS_CREDENTIAL_FILE: it defaults under + # SQI_WORKER_DATA_DIR, which is what makes this worker's enrolled + # credential land in its own fresh, per-run data directory. + export SQI_WORKER_NATS_JOIN_TOKEN_FILE="$JOIN_TOKEN_FILE" + export SQI_WORKER_NATS_SERVER_URL="$BASE_URL" + ;; + tls) + export SQI_WORKER_NATS_TLS_CA_FILE="${CERT_DIR}/ca.crt" + export SQI_WORKER_NATS_SERVER_TLS_CA_FILE="${CERT_DIR}/ca.crt" + ;; +esac +"$WORKER_BIN" start >"$WORKER_LOG" 2>&1 & WORKER_PID=$! # Poll GET /api/v1/workers until our worker is online. @@ -348,7 +369,7 @@ for _ in $(seq 1 150); do log_tail "worker log" "$WORKER_LOG" fail "sqi-worker exited before coming online" fi - workers_json="$(curl -s "${BASE_URL}/api/v1/workers" 2>/dev/null || true)" + workers_json="$(curl -s ${CURL_CA} "${BASE_URL}/api/v1/workers" 2>/dev/null || true)" if [ "$HAVE_JQ" -eq 1 ]; then WORKER_ID="$(printf '%s' "$workers_json" \ | jq -r '[.items[]? | select(.status=="online") | .id][0] // empty')" @@ -385,7 +406,7 @@ YAML )" submit_url="${BASE_URL}/api/v1/jobs?farm_id=${FARM_ID}&queue_id=${QUEUE_ID}&owner=smoke" -job_json="$(curl -s -X POST "$submit_url" \ +job_json="$(curl -s ${CURL_CA} -X POST "$submit_url" \ -H 'Content-Type: application/x-yaml' \ --data-binary "$JOB_YAML")" JOB_ID="$(json_get "$job_json" id)" @@ -396,7 +417,7 @@ log "submitted job ${JOB_ID} (sentinel: ${SENTINEL})" TASK_ID="" for _ in $(seq 1 100); do - tasks_json="$(curl -s "${BASE_URL}/api/v1/jobs/${JOB_ID}/tasks" 2>/dev/null || true)" + tasks_json="$(curl -s ${CURL_CA} "${BASE_URL}/api/v1/jobs/${JOB_ID}/tasks" 2>/dev/null || true)" TASK_ID="$(first_task_id "$tasks_json")" [ -n "$TASK_ID" ] && break sleep 0.2 @@ -413,19 +434,32 @@ log "task ${TASK_ID}" WS_OK="skip" if [ -n "$WS_PYTHON" ]; then WS_OK="armed" + # In TLS mode the gateway is wss:// and the client must trust the farm CA. + # WS_CA is passed as a 5th argv and is the empty string in the other modes. WS_URL="ws://${HTTP_ADDR}/api/v1/ws" + WS_CA="" + if [ "$mode" = "tls" ]; then + WS_URL="wss://${HTTP_ADDR}/api/v1/ws" + WS_CA="${CERT_DIR}/ca.crt" + fi log "arming WebSocket subscriber via ${WS_PYTHON} (subject tasks/${TASK_ID}/logs)" - "$WS_PYTHON" - "$WS_URL" "tasks/${TASK_ID}/logs" "$SENTINEL" "$WS_READY" <<'PY' >"$WS_OUT" 2>&1 & + "$WS_PYTHON" - "$WS_URL" "tasks/${TASK_ID}/logs" "$SENTINEL" "$WS_READY" "$WS_CA" <<'PY' >"$WS_OUT" 2>&1 & import json +import ssl import sys import time from websockets.sync.client import connect ws_url, subject, sentinel, ready_path = sys.argv[1:5] +ca_file = sys.argv[5] if len(sys.argv) > 5 else "" deadline = time.monotonic() + 45.0 -with connect(ws_url, open_timeout=10) as conn: +kwargs = {"open_timeout": 10} +if ca_file: + kwargs["ssl"] = ssl.create_default_context(cafile=ca_file) + +with connect(ws_url, **kwargs) as conn: conn.send(json.dumps({ "type": "subscribe", "subject": subject, @@ -493,7 +527,7 @@ fi JOB_STATUS="" for _ in $(seq 1 200); do - job_json="$(curl -s "${BASE_URL}/api/v1/jobs/${JOB_ID}" 2>/dev/null || true)" + job_json="$(curl -s ${CURL_CA} "${BASE_URL}/api/v1/jobs/${JOB_ID}" 2>/dev/null || true)" JOB_STATUS="$(json_get "$job_json" status)" case "$JOB_STATUS" in completed|failed|canceled) break ;; @@ -511,7 +545,7 @@ log "job completed" REST_OK=0 for _ in $(seq 1 75); do - logs_json="$(curl -s "${BASE_URL}/api/v1/tasks/${TASK_ID}/logs" 2>/dev/null || true)" + logs_json="$(curl -s ${CURL_CA} "${BASE_URL}/api/v1/tasks/${TASK_ID}/logs" 2>/dev/null || true)" if logs_contain "$logs_json" "$SENTINEL"; then REST_OK=1 break @@ -593,7 +627,7 @@ steps: YAML )" -expr_job_json="$(curl -s -X POST "$submit_url" \ +expr_job_json="$(curl -s ${CURL_CA} -X POST "$submit_url" \ -H 'Content-Type: application/x-yaml' \ --data-binary "$EXPR_JOB_YAML")" EXPR_JOB_ID="$(json_get "$expr_job_json" id)" @@ -605,7 +639,7 @@ log "submitted EXPR job ${EXPR_JOB_ID} (sentinel: ${EXPR_SENTINEL})" EXPR_TASK_ID="" for _ in $(seq 1 100); do - expr_tasks_json="$(curl -s "${BASE_URL}/api/v1/jobs/${EXPR_JOB_ID}/tasks" 2>/dev/null || true)" + expr_tasks_json="$(curl -s ${CURL_CA} "${BASE_URL}/api/v1/jobs/${EXPR_JOB_ID}/tasks" 2>/dev/null || true)" EXPR_TASK_ID="$(first_task_id "$expr_tasks_json")" [ -n "$EXPR_TASK_ID" ] && break sleep 0.2 @@ -615,7 +649,7 @@ log "EXPR task ${EXPR_TASK_ID}" EXPR_JOB_STATUS="" for _ in $(seq 1 200); do - expr_job_json="$(curl -s "${BASE_URL}/api/v1/jobs/${EXPR_JOB_ID}" 2>/dev/null || true)" + expr_job_json="$(curl -s ${CURL_CA} "${BASE_URL}/api/v1/jobs/${EXPR_JOB_ID}" 2>/dev/null || true)" EXPR_JOB_STATUS="$(json_get "$expr_job_json" status)" case "$EXPR_JOB_STATUS" in completed|failed|canceled) break ;; @@ -631,7 +665,7 @@ log "EXPR job completed" EXPR_OK=0 for _ in $(seq 1 75); do - expr_logs_json="$(curl -s "${BASE_URL}/api/v1/tasks/${EXPR_TASK_ID}/logs" 2>/dev/null || true)" + expr_logs_json="$(curl -s ${CURL_CA} "${BASE_URL}/api/v1/tasks/${EXPR_TASK_ID}/logs" 2>/dev/null || true)" if logs_contain "$expr_logs_json" "$EXPR_EXPECTED"; then EXPR_OK=1 break @@ -666,15 +700,20 @@ exit 0 # starts, and the last banner printed is the one that broke. log "==================================================================" -log "MODE 1/2: broker auth OFF -- the default path, unmodified" +log "MODE 1/3: broker auth OFF, no TLS -- the default path, unmodified" log "==================================================================" ( run_smoke_flow noauth ) log "==================================================================" -log "MODE 2/2: broker auth ON, worker enrolled via a join token" +log "MODE 2/3: broker auth ON, worker enrolled via a join token" log "==================================================================" ( run_smoke_flow brokerauth ) log "==================================================================" -log "SMOKE TEST PASSED IN BOTH MODES" +log "MODE 3/3: TLS ON -- HTTPS REST, WSS gateway, TLS broker" +log "==================================================================" +( run_smoke_flow tls ) + +log "==================================================================" +log "SMOKE TEST PASSED IN ALL THREE MODES" log "==================================================================" diff --git a/test/integration/broker_auth_test.go b/test/integration/broker_auth_test.go index ce79ad61..6b220e84 100644 --- a/test/integration/broker_auth_test.go +++ b/test/integration/broker_auth_test.go @@ -28,6 +28,7 @@ import ( "log/slog" "net/http" "os" + "path/filepath" "testing" "time" @@ -115,6 +116,15 @@ func startBrokerAuthServer(t *testing.T, sqlitePath string, mutate func(*server. } ts := &testServer{HTTPAddr: httpAddr, NATSAddr: natsAddr, cancel: cancel, done: done} + // Derive the transport from the config this server was actually given, so + // every shared helper (apiURL, seedFarmAndQueue, pollForOnlineWorker, the + // readiness probe below) speaks the right scheme with no TLS-specific + // variants. `sqi-server tls init` writes ca.crt beside the server cert, + // which is what the TLS callers here use. + if cfg.HTTPTLS.Enabled { + ts.Scheme = "https" + ts.Client = tlsClient(t, filepath.Dir(cfg.HTTPTLS.CertFile)) + } t.Cleanup(func() { cancel() select { @@ -127,7 +137,11 @@ func startBrokerAuthServer(t *testing.T, sqlitePath string, mutate func(*server. if !waitForTCP(t, httpAddr, 10*time.Second) { t.Fatal("startBrokerAuthServer: HTTP server did not start listening") } - if !waitForReadyz(t, httpAddr, 10*time.Second) { + // Readiness must be probed over whatever scheme this server is actually + // serving: an https:// listener answers a plaintext GET with a 400, which + // would look like "never became ready" forever. A TLS caller stamps + // ts.Scheme/ts.Client before this runs (see startTLSServer in tls_test.go). + if !waitForReadyzClient(t, clientFor(ts), apiURL(ts, "/readyz"), 10*time.Second) { t.Fatal("startBrokerAuthServer: server did not become ready") } return ts diff --git a/test/integration/discovery_test.go b/test/integration/discovery_test.go new file mode 100644 index 00000000..7d55365e --- /dev/null +++ b/test/integration/discovery_test.go @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build integration + +package integration + +// End-to-end coverage of the mDNS discovery path over REAL multicast. +// +// Everything else in this suite sets DiscoveryEnabled: false, so until these +// tests existed the discovery path had unit coverage on both halves and +// nothing joining them: the server advertised TXT records that no test ever +// received, and the worker parsed TXT records that no test ever sent. The +// specific gap that mattered was the TLS records — the server advertised +// nats_tls=1 and, for a while, nothing on the worker read it at all. +// +// These tests use real multicast on a real interface. See multicast_test.go +// for the preflight and for why a foreign sqi-server on the network makes the +// real-binary test refuse to run. + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/uberware/sqi/internal/config" + serverdiscovery "github.com/uberware/sqi/internal/discovery" + "github.com/uberware/sqi/internal/server" + workerconfig "github.com/uberware/sqi/internal/worker/config" + workerdiscovery "github.com/uberware/sqi/internal/worker/discovery" +) + +// advertise starts a real mDNS responder describing a server with the given +// TLS posture, and returns its instance name. +func advertise(t *testing.T, httpTLS, natsTLS bool) string { + t.Helper() + instance := instanceName(t, "sqi-disco") + resp, err := serverdiscovery.New(serverdiscovery.Config{ + Enabled: true, + InstanceName: instance, + HTTPAddr: "127.0.0.1:18080", + NATSAddr: "127.0.0.1:14222", + HTTPTLS: httpTLS, + NATSTLS: natsTLS, + // Loopback only: a test must not announce a service on the network it + // happens to be running on. + Interfaces: loopbackIfaces(), + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatalf("responder New: %v", err) + } + if err := resp.Start(context.Background()); err != nil { + t.Fatalf("responder Start: %v", err) + } + t.Cleanup(resp.Shutdown) + return instance +} + +// browseOnce browses for this test's advertisement and returns the result. +func browseOnce(t *testing.T) workerdiscovery.Result { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + found, err := workerdiscovery.Browse(ctx, 10*time.Second, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatalf("Browse: %v", err) + } + return found +} + +// TestDiscovery_TLSRecordsCrossTheWire is the join the unit tests could not +// make: the server's own config decides the TXT records, they travel over +// real multicast, and the worker's parser reads them back. +func TestDiscovery_TLSRecordsCrossTheWire(t *testing.T) { + requireMulticast(t) + noForeignServer(t) + + tests := []struct { + name string + httpTLS, natsTLS bool + }{ + {"plaintext farm", false, false}, + {"tls on both listeners", true, true}, + {"broker tls only", false, true}, + {"api tls only", true, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instance := advertise(t, tt.httpTLS, tt.natsTLS) + found := browseOnce(t) + + // Guard against having discovered something else entirely: a + // wrong-server result would otherwise look like a parsing bug. + if found.InstanceName != instance { + t.Fatalf("discovered %q, want this test's %q", found.InstanceName, instance) + } + if found.NATSTLS != tt.natsTLS { + t.Errorf("NATSTLS = %v, want %v (nats_tls TXT record did not survive the wire)", found.NATSTLS, tt.natsTLS) + } + if found.HTTPTLS != tt.httpTLS { + t.Errorf("HTTPTLS = %v, want %v (tls TXT record did not survive the wire)", found.HTTPTLS, tt.httpTLS) + } + if !strings.HasPrefix(found.NATSURL, "nats://") { + t.Errorf("NATSURL = %q, want a nats:// URL", found.NATSURL) + } + }) + } +} + +// TestDiscovery_AdvertisedBrokerTLSReachesWorkerConfig runs the whole consumer +// chain over real multicast: server config → TXT records → wire → parse → +// the worker's TLS decision. +// +// This is the regression that matters. The records were advertised for a +// while with nothing reading them, so a worker discovering a TLS farm +// attempted plaintext and failed with an error naming neither the cause nor +// the setting that fixes it. +func TestDiscovery_AdvertisedBrokerTLSReachesWorkerConfig(t *testing.T) { + requireMulticast(t) + noForeignServer(t) + + t.Run("auto enables TLS from the advertisement alone", func(t *testing.T) { + advertise(t, true, true) + found := browseOnce(t) + + // A worker with NOTHING configured: no URL, no CA, no tls_enabled. + // Only the advertisement can turn TLS on here. + cfg := workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto} + if cfg.UseTLS(false) { + t.Fatal("fixture is wrong: this config already wanted TLS before discovery") + } + workerdiscovery.ApplyTLS(context.Background(), &cfg, found, slog.New(slog.DiscardHandler)) + if !cfg.UseTLS(false) { + t.Error("a discovered TLS-required broker did not enable TLS under auto") + } + }) + + t.Run("plaintext farm leaves the worker plaintext", func(t *testing.T) { + advertise(t, false, false) + found := browseOnce(t) + + cfg := workerconfig.NATSConfig{TLSEnabled: workerconfig.TLSAuto} + workerdiscovery.ApplyTLS(context.Background(), &cfg, found, slog.New(slog.DiscardHandler)) + if cfg.UseTLS(false) { + t.Error("a plaintext farm turned TLS on; the default deployment would break") + } + }) +} + +// TestDiscovery_RealBinaryFindsItsServerOverMDNS is the full stack: a real +// sqi-server advertising over mDNS with TLS on both listeners, and a real +// sqi-worker subprocess given NO nats.url at all, which has to discover the +// server, act on nats_tls, and register. +func TestDiscovery_RealBinaryFindsItsServerOverMDNS(t *testing.T) { + // This is the ONE test here that opens a listener beyond loopback, so it + // runs only when discovery testing was asked for explicitly — `make + // test-discovery` or the CI job — never as a side effect of `make + // test-integration`. + // + // It cannot avoid the listener. The advertisement carries this machine's + // HOSTNAME (entryToResult prefers entry.HostName), so the worker dials that + // name whatever interface the announcement went out on, and a + // loopback-bound broker is unreachable there. Verified by trying: bound to + // 127.0.0.1 the worker discovers the server and then fails with "no servers + // available". Changing the product to advertise a loopback literal would be + // bending production behavior to suit a test. + if !multicastRequired() { + t.Skip("binds the test broker to all interfaces; run `make test-discovery` to include it") + } + requireMulticast(t) + noForeignServer(t) + + // Two consequences of the advertisement carrying a hostname, both of them + // what a real deployment does anyway: + // + // 1. The broker must bind all interfaces, per the note above. + // 2. The certificate must cover the name OR the address the worker ends up + // dialing. Advertisements here are loopback-only, so ".local" + // is answered by nothing the system resolver queries and the worker + // falls back to the advertised IP (see discovery.resolveHost) — which + // is why the loopback addresses below matter as much as the hostname. + host, err := os.Hostname() + if err != nil { + t.Fatalf("os.Hostname: %v", err) + } + host = strings.TrimSuffix(host, ".local") + dir := tlsMaterialFor(t, []string{host, host + ".local", "localhost", "127.0.0.1", "::1"}) + + dbPath := filepath.Join(t.TempDir(), "sqi.db") + joinToken := seedJoinToken(t, dbPath, "discovery-e2e") + + certFile := filepath.Join(dir, "server.crt") + keyFile := filepath.Join(dir, "server.key") + instance := instanceName(t, "sqi-e2e") + + ts := startBrokerAuthServer(t, dbPath, func(cfg *server.Config) { + cfg.HTTPTLS = config.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + cfg.NATSTLS = config.NATSTLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + cfg.NATSAuthEnrollmentEndpointEnabled = true + // Reachable at the advertised hostname, per (1) above. + cfg.NATSAddr = strings.Replace(cfg.NATSAddr, "127.0.0.1", "0.0.0.0", 1) + // The point of this test: advertise, so the worker can find us. + cfg.DiscoveryEnabled = true + cfg.DiscoveryInstanceName = instance + cfg.DiscoveryInterfaces = loopbackIfaces() + }) + + farmID, queueID := seedFarmAndQueue(t, ts) + caFile := filepath.Join(dir, "ca.crt") + + // No SQI_WORKER_NATS_URL: the worker must find the broker over mDNS, and + // the discovered nats_tls record is what tells it the transport. + startRealWorkerNoWait(t, ts, farmID, queueID, nil, []string{ + // Blank out the harness default so the worker has no URL to fall back + // on: applyNATSEnv ignores an empty value, so nats.url stays unset and + // mDNS is the only way this worker can find its broker. + "SQI_WORKER_NATS_URL=", + "SQI_WORKER_DISCOVERY_ENABLE_MDNS=true", + "SQI_WORKER_DISCOVERY_MDNS_TIMEOUT=15s", + "SQI_WORKER_NATS_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_SERVER_URL=https://" + ts.HTTPAddr, + "SQI_WORKER_NATS_SERVER_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_JOIN_TOKEN=" + joinToken, + }, true) + + if id := findOnlineWorker(t, ts, farmID, 60*time.Second); id == "" { + t.Fatal("worker never came online: mDNS discovery, TLS from the advertisement, or enrollment failed") + } +} diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 5813c9df..75ba64e2 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -64,7 +64,11 @@ type workerListResp struct { // apiURL builds a full URL for the test server. func apiURL(ts *testServer, path string) string { - return "http://" + ts.HTTPAddr + path + scheme := ts.Scheme + if scheme == "" { + scheme = "http" + } + return scheme + "://" + ts.HTTPAddr + path } // mustDoJSON performs an HTTP request and decodes the JSON response body into @@ -72,6 +76,14 @@ func apiURL(ts *testServer, path string) string { // expected status code set. func mustDoJSON(t *testing.T, method, url string, body []byte, contentType string, expectStatus int, dst any) { t.Helper() + mustDoJSONClient(t, httpClient, method, url, body, contentType, expectStatus, dst) +} + +// mustDoJSONClient is [mustDoJSON] with a caller-supplied client, for a server +// whose certificate the package-default client cannot verify. Same shape as +// waitForReadyzClient in harness_test.go. +func mustDoJSONClient(t *testing.T, client *http.Client, method, url string, body []byte, contentType string, expectStatus int, dst any) { + t.Helper() var reqBody *bytes.Reader if body != nil { @@ -88,7 +100,7 @@ func mustDoJSON(t *testing.T, method, url string, body []byte, contentType strin req.Header.Set("Content-Type", contentType) } - resp, err := httpClient.Do(req) + resp, err := client.Do(req) if err != nil { t.Fatalf("mustDoJSON: %s %s: %v", method, url, err) } @@ -116,7 +128,7 @@ func submitJob(t *testing.T, ts *testServer, farmID, queueID string) string { t.Helper() url := fmt.Sprintf("%s?farm_id=%s&queue_id=%s&owner=test", apiURL(ts, "/api/v1/jobs"), farmID, queueID) var resp jobResp - mustDoJSON(t, http.MethodPost, url, []byte(minimalJobYAML), "application/x-yaml", http.StatusCreated, &resp) + mustDoJSONClient(t, clientFor(ts), http.MethodPost, url, []byte(minimalJobYAML), "application/x-yaml", http.StatusCreated, &resp) if resp.ID == "" { t.Fatal("submitJob: server returned empty job ID") } @@ -136,7 +148,7 @@ func pollJobStatus(t *testing.T, ts *testServer, jobID string, targets []string, deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { var resp jobResp - mustDoJSON(t, http.MethodGet, apiURL(ts, "/api/v1/jobs/"+jobID), nil, "", http.StatusOK, &resp) + mustDoJSONClient(t, clientFor(ts), http.MethodGet, apiURL(ts, "/api/v1/jobs/"+jobID), nil, "", http.StatusOK, &resp) lastStatus = resp.Status if targetSet[resp.Status] { return resp.Status @@ -177,7 +189,7 @@ func seedFarmAndQueue(t *testing.T, ts *testServer) (farmID, queueID string) { var farmResp struct { ID string `json:"id"` } - mustDoJSON(t, http.MethodPost, apiURL(ts, "/api/v1/farms"), farmBody, "application/json", http.StatusCreated, &farmResp) + mustDoJSONClient(t, clientFor(ts), http.MethodPost, apiURL(ts, "/api/v1/farms"), farmBody, "application/json", http.StatusCreated, &farmResp) if farmResp.ID == "" { t.Fatal("seedFarmAndQueue: server returned empty farm ID") } @@ -193,7 +205,7 @@ func seedFarmAndQueue(t *testing.T, ts *testServer) (farmID, queueID string) { var queueResp struct { ID string `json:"id"` } - mustDoJSON(t, http.MethodPost, apiURL(ts, "/api/v1/queues"), queueBody, "application/json", http.StatusCreated, &queueResp) + mustDoJSONClient(t, clientFor(ts), http.MethodPost, apiURL(ts, "/api/v1/queues"), queueBody, "application/json", http.StatusCreated, &queueResp) if queueResp.ID == "" { t.Fatal("seedFarmAndQueue: server returned empty queue ID") } diff --git a/test/integration/harness_test.go b/test/integration/harness_test.go index 8af7eb93..4cc5f018 100644 --- a/test/integration/harness_test.go +++ b/test/integration/harness_test.go @@ -75,6 +75,15 @@ type testServer struct { // NATSAddr is the full "host:port" address the embedded NATS server // is listening on. Workers connect to "nats://" + NATSAddr. NATSAddr string + // Scheme is "http" or "https", matching whether this server terminates + // TLS. Empty means "http". apiURL and every helper built on it read this, + // so a TLS server needs no parallel set of helpers. + Scheme string + + // Client performs requests against this server. Nil means the package's + // default plaintext client; a TLS server sets one trusting its farm CA. + Client *http.Client + // DBPath is the SQLite file this server was started against. Set by // constructors that know it, empty otherwise. A caller that needs to // inspect store state the REST API does not expose (e.g. confirming no @@ -124,7 +133,13 @@ func startServer(t *testing.T) *testServer { WorkerTimeout: 30 * time.Second, HeartbeatSweepInterval: 15 * time.Second, }, - // mDNS disabled: multicast is not available in most CI environments. + // mDNS disabled: these tests reach the server by an explicit address, so + // advertising would add a real network side effect for no coverage. + // + // NOT because multicast is unavailable — this comment used to say that, + // and it was never tested. It is tested now: discovery_test.go proves an + // mDNS round trip on a real interface, and `make test-discovery` fails + // rather than skips when one is not possible. DiscoveryEnabled: false, } @@ -184,6 +199,15 @@ func startServer(t *testing.T) *testServer { return ts } +// clientFor returns the HTTP client to use against ts: its own when it has one +// (a TLS server trusting its farm CA), the package default otherwise. +func clientFor(ts *testServer) *http.Client { + if ts != nil && ts.Client != nil { + return ts.Client + } + return httpClient +} + // waitForTCP dials addr in a polling loop until the connection succeeds or // timeout expires. Returns true if the port became reachable. func waitForTCP(tb testing.TB, addr string, timeout time.Duration) bool { @@ -206,8 +230,14 @@ func waitForTCP(tb testing.TB, addr string, timeout time.Duration) bool { // stream provisioning) are fully up before tests start. func waitForReadyz(tb testing.TB, httpAddr string, timeout time.Duration) bool { tb.Helper() - client := &http.Client{Timeout: 2 * time.Second} - url := "http://" + httpAddr + "/readyz" + return waitForReadyzClient(tb, &http.Client{Timeout: 2 * time.Second}, "http://"+httpAddr+"/readyz", timeout) +} + +// waitForReadyzClient is waitForReadyz against a caller-supplied client and +// base URL. A TLS-terminated server needs both: an https:// scheme, and a +// client that trusts the farm CA. See tls_test.go. +func waitForReadyzClient(tb testing.TB, client *http.Client, url string, timeout time.Duration) bool { + tb.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) diff --git a/test/integration/multicast_test.go b/test/integration/multicast_test.go new file mode 100644 index 00000000..df96b46f --- /dev/null +++ b/test/integration/multicast_test.go @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build integration + +package integration + +// Multicast capability detection for the mDNS discovery tests. +// +// The rest of this suite disables mDNS, on the long-standing assumption that +// "multicast is not available in most CI environments". That assumption was +// never tested. It is tested here: requireMulticast actually performs a +// round trip and reports what happened, rather than guessing from the +// environment. +// +// These tests do NOT transmit on the network they are running on. Every +// advertisement is restricted to loopback (see loopbackIfaces), which a browser +// listening on all interfaces still receives — verified, not assumed. That +// keeps a test run from announcing a service on an office LAN, and it is an +// invariant rather than a preference: where loopback cannot carry multicast the +// tests refuse to run rather than falling back to a real interface. +// +// Because a skipped test verifies nothing, SQI_TEST_REQUIRE_MULTICAST=1 turns +// the skip into a failure. `make test-discovery` and the CI job set it, so a +// runner that silently loses multicast fails the build instead of quietly +// covering less than it claims. + +import ( + "context" + "log/slog" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + serverdiscovery "github.com/uberware/sqi/internal/discovery" + workerdiscovery "github.com/uberware/sqi/internal/worker/discovery" +) + +// multicastRequired reports whether a skip should be treated as a failure. +func multicastRequired() bool { + v := strings.ToLower(strings.TrimSpace(os.Getenv("SQI_TEST_REQUIRE_MULTICAST"))) + return v == "1" || v == "true" || v == "yes" +} + +// loopbackIfaces returns the multicast-capable loopback interfaces. +// +// Advertising only on these is what keeps a test run off the real network. It +// works because multicast sent on loopback is delivered to anything listening +// on loopback, including a resolver bound to every interface — which is what +// the production worker uses, so the real-binary test needs no special casing. +// +// Linux `lo` does not carry the MULTICAST flag by default, which is why the CI +// job enables it — and having the flag is necessary but not sufficient, see +// advertisable. docs/development.md carries both remediations. +func loopbackIfaces() []net.Interface { + ifs, err := net.Interfaces() + if err != nil { + return nil + } + var out []net.Interface + for _, i := range ifs { + if i.Flags&net.FlagLoopback != 0 && i.Flags&net.FlagMulticast != 0 && i.Flags&net.FlagUp != 0 { + out = append(out, i) + } + } + return out +} + +// requireMulticast proves an mDNS round trip works on this host before a test +// depends on one, by advertising a throwaway service on loopback and browsing +// for it. +// +// It exists so a discovery test fails for the reason it actually failed: a +// broken responder looks identical to a host that cannot do multicast at all, +// and without this preflight every such failure would be investigated twice. +func requireMulticast(t *testing.T) { + t.Helper() + + lo := loopbackIfaces() + if len(lo) == 0 { + skipOrFail(t, "loopback has no MULTICAST flag, and these tests refuse to "+ + "advertise on a real interface; on Linux run: sudo ip link set lo multicast on") + return + } + + if !anyAdvertisable(lo) { + skipOrFail(t, "loopback carries loopback addresses only, and zeroconf "+ + "discards those when it builds the advertisement's address records, "+ + "so a loopback-only advertisement cannot register at all; on Linux "+ + "run: sudo ip -6 addr add fe80::1/64 dev lo") + return + } + + logger := slog.New(slog.DiscardHandler) + probe, err := serverdiscovery.New(serverdiscovery.Config{ + Enabled: true, + InstanceName: instanceName(t, "sqi-probe"), + HTTPAddr: "127.0.0.1:1", + NATSAddr: "127.0.0.1:2", + Interfaces: lo, + }, logger) + if err != nil { + t.Fatalf("multicast probe: responder New: %v", err) + } + if err := probe.Start(context.Background()); err != nil { + skipOrFail(t, "multicast probe: responder could not start: "+err.Error()) + return + } + defer probe.Shutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + if _, err := workerdiscovery.Browse(ctx, 6*time.Second, logger); err != nil { + skipOrFail(t, "multicast probe: no mDNS round trip over loopback on this host ("+ + err.Error()+")") + } +} + +// anyAdvertisable reports whether any of these interfaces can carry an +// advertisement. +func anyAdvertisable(ifaces []net.Interface) bool { + for _, i := range ifaces { + addrs, err := i.Addrs() + if err != nil { + continue + } + if advertisable(addrs) { + return true + } + } + return false +} + +// advertisable mirrors the address filter zeroconf applies when it builds an +// advertisement, so a host that cannot register says so in its own terms. +// +// zeroconf fills the A/AAAA records from the interface's own addresses and +// drops every loopback one (addrsForInterface: `!ipnet.IP.IsLoopback()`), then +// fails with "Could not determine host IP addresses" if nothing is left. Linux +// `lo` carries only 127.0.0.1 and ::1, so registering on loopback alone fails +// there outright; macOS lo0 also carries fe80::1, which is why the same test +// passes on a Mac. Without this check that shows up as an error three layers +// down which says nothing about loopback, and the multicast flag — which is +// genuinely fine — is the first thing anyone suspects. +func advertisable(addrs []net.Addr) bool { + for _, a := range addrs { + if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + return true + } + } + return false +} + +// skipOrFail skips, unless multicast was declared required. +func skipOrFail(t *testing.T, reason string) { + t.Helper() + if multicastRequired() { + t.Fatalf("%s — and SQI_TEST_REQUIRE_MULTICAST is set, so this is a failure, not a skip", reason) + } + t.Skip(reason) +} + +// maxInstanceLabel is the DNS label limit a DNS-SD instance name must fit in. +// Exceeding it does not error: the responder starts and is simply never +// discoverable, which presents as "multicast does not work on this host". +// This cost an hour of looking in the wrong place; the cap is enforced in +// instanceName so no caller has to remember it. +const maxInstanceLabel = 63 + +// instanceName builds a unique, valid mDNS instance name for this test run. +// +// Uniqueness matters because these tests advertise on a real network: two runs +// on one machine, or two machines on one LAN, must not answer for each other. +func instanceName(t *testing.T, prefix string) string { + t.Helper() + safe := func(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + b.WriteRune(r) + default: + b.WriteByte('-') + } + } + return b.String() + } + + // t.TempDir() is unique per test; its last two elements carry the random + // component and the per-test counter, which is all the uniqueness needed. + dir := t.TempDir() + unique := safe(filepath.Base(filepath.Dir(dir)) + "-" + filepath.Base(dir)) + if len(unique) > 24 { + unique = unique[len(unique)-24:] + } + + name := safe(prefix) + "-" + unique + if len(name) > maxInstanceLabel { + name = name[len(name)-maxInstanceLabel:] + } + return strings.Trim(name, "-") +} + +// noForeignServer skips when something other than this test is already +// advertising _sqi._tcp. +// +// The tests never advertise beyond loopback, but they still BROWSE on every +// interface — the production worker does, and the real-binary test runs the +// production worker. So a live sqi-server on the same LAN can still be +// discovered, and the test cannot tell a colleague's production broker from its +// own before connecting to it. Refusing to run is the only safe response. +func noForeignServer(t *testing.T) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + found, err := workerdiscovery.Browse(ctx, 2*time.Second, slog.New(slog.DiscardHandler)) + if err == nil { + t.Skipf("another sqi-server is advertising on this network (%s at %s); "+ + "refusing to run a real-mDNS test that could discover it", + found.InstanceName, found.NATSURL) + } +} + +// TestMulticastRequired_FlagParsing pins the switch that decides whether a +// missing capability is a skip or a failure. +// +// It is worth a test of its own because everything else in this file trusts +// it: get this wrong in the "skip" direction and `make test-discovery` and the +// CI job go green while running nothing, which is the exact failure the flag +// exists to prevent. +func TestMulticastRequired_FlagParsing(t *testing.T) { + for _, v := range []string{"1", "true", "TRUE", "yes", " 1 "} { + t.Setenv("SQI_TEST_REQUIRE_MULTICAST", v) + if !multicastRequired() { + t.Errorf("SQI_TEST_REQUIRE_MULTICAST=%q did not require multicast", v) + } + } + for _, v := range []string{"", "0", "false", "no", "off"} { + t.Setenv("SQI_TEST_REQUIRE_MULTICAST", v) + if multicastRequired() { + t.Errorf("SQI_TEST_REQUIRE_MULTICAST=%q required multicast", v) + } + } +} + +// TestInstanceName_FitsTheDNSLabelLimit guards the trap that cost real time +// while these tests were being written: an over-long instance name does not +// error, it just makes the responder undiscoverable, which is indistinguishable +// from the host having no multicast at all. +func TestInstanceName_FitsTheDNSLabelLimit(t *testing.T) { + got := instanceName(t, "a-deliberately-long-prefix-that-would-overflow-the-label-limit-on-its-own") + if len(got) > maxInstanceLabel { + t.Errorf("instance name is %d bytes (%q), want <= %d", len(got), got, maxInstanceLabel) + } + if got == "" { + t.Error("instance name is empty") + } + + // Uniqueness comes from t.TempDir(), which differs per call, so comparing + // two calls proves nothing. What matters is that the unique part SURVIVES + // the length cap — truncation keeps the tail for exactly that reason, and + // a change that trimmed the other end would make every over-long name + // collide on its shared prefix. + long := instanceName(t, strings.Repeat("x", 200)) + short := instanceName(t, "p") + if len(long) != maxInstanceLabel { + t.Errorf("an over-long name is %d bytes, want it capped at exactly %d", len(long), maxInstanceLabel) + } + if !strings.HasSuffix(short, strings.TrimPrefix(short, "p-")) { + t.Errorf("unique suffix missing from %q", short) + } + if long == instanceName(t, strings.Repeat("y", 200)) { + t.Error("two over-long names collided after truncation; the unique tail was trimmed") + } +} + +// TestAdvertisable_MatchesZeroconfAddressFilter pins the check that tells a +// Linux host why a loopback-only advertisement cannot register. +// +// The two loopback rows are the whole point: they are what macOS and Linux +// actually put on their loopback interface, and only one of them can carry an +// advertisement. +func TestAdvertisable_MatchesZeroconfAddressFilter(t *testing.T) { + addr := func(cidr string) net.Addr { + ip, n, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatalf("ParseCIDR(%q): %v", cidr, err) + } + return &net.IPNet{IP: ip, Mask: n.Mask} + } + + for _, tc := range []struct { + name string + addrs []net.Addr + want bool + }{ + {"linux lo", []net.Addr{addr("127.0.0.1/8"), addr("::1/128")}, false}, + {"macos lo0", []net.Addr{addr("127.0.0.1/8"), addr("::1/128"), addr("fe80::1/64")}, true}, + {"no addresses", nil, false}, + {"routable interface", []net.Addr{addr("192.168.1.10/24")}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := advertisable(tc.addrs); got != tc.want { + t.Errorf("advertisable() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/test/integration/tls_test.go b/test/integration/tls_test.go new file mode 100644 index 00000000..0d1a5a35 --- /dev/null +++ b/test/integration/tls_test.go @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build integration + +package integration + +// TestTLSEndToEnd proves the whole TLS path with real binaries and real +// generated certificates: sqi-server serves HTTPS, its embedded broker +// requires TLS, and a real sqi-worker subprocess enrolls over HTTPS with a +// join token and then connects to the TLS broker. +// +// Enrollment is the part that unit tests cannot prove is sufficient. It runs +// over REST BEFORE the worker holds any broker credential, so it is the first +// thing a farm does and the first thing that breaks if the worker cannot +// trust the server's certificate. +// +// The counterpart is TestTLSWorkerWithoutCAIsRefused: the same server, a +// worker given no CA, which must never register. Without it this suite could +// pass while TLS was quietly optional. + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/uberware/sqi/internal/certgen" + "github.com/uberware/sqi/internal/config" + "github.com/uberware/sqi/internal/server" +) + +// tlsMaterial generates a farm CA and a loopback server certificate into a +// temp directory, returning the directory. It uses the same internal/certgen +// package that `sqi-server tls init` uses, so this exercises the shipped +// generator rather than a test-only substitute. +func tlsMaterial(t *testing.T) string { + t.Helper() + return tlsMaterialFor(t, []string{"localhost", "127.0.0.1", "::1"}) +} + +// tlsMaterialFor is [tlsMaterial] with an explicit SAN list, for a server that +// has to be reachable by a name other than loopback — see the mDNS discovery +// tests, where the advertisement carries this machine's hostname. +func tlsMaterialFor(t *testing.T, sans []string) string { + t.Helper() + dir := t.TempDir() + ca, err := certgen.NewCA("integration farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + leaf, err := ca.NewServerCert(sans, 365*24*time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "server", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + return dir +} + +// tlsMaterialWithClient generates the farm CA, a server certificate and a +// worker CLIENT certificate in one pass, returning the directory and the +// client keypair's paths. +// +// It mints all three together because `sqi-server tls init` refuses to +// overwrite an existing CA, so a second invocation cannot issue a client +// certificate against a CA it already wrote. +func tlsMaterialWithClient(t *testing.T, workerID string) (dir, certFile, keyFile string) { + t.Helper() + dir = t.TempDir() + ca, err := certgen.NewCA("integration farm CA", 10*365*24*time.Hour) + if err != nil { + t.Fatalf("NewCA: %v", err) + } + if err := certgen.WriteCA(dir, ca); err != nil { + t.Fatalf("WriteCA: %v", err) + } + server, err := ca.NewServerCert([]string{"localhost", "127.0.0.1", "::1"}, 365*24*time.Hour) + if err != nil { + t.Fatalf("NewServerCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "server", server); err != nil { + t.Fatalf("WriteLeaf(server): %v", err) + } + client, err := ca.NewClientCert(workerID, 365*24*time.Hour) + if err != nil { + t.Fatalf("NewClientCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "client-"+workerID, client); err != nil { + t.Fatalf("WriteLeaf(client): %v", err) + } + return dir, filepath.Join(dir, "client-"+workerID+".crt"), filepath.Join(dir, "client-"+workerID+".key") +} + +// tlsClient returns an HTTP client trusting only the CA in dir. +func tlsClient(t *testing.T, dir string) *http.Client { + t.Helper() + pem, err := os.ReadFile(filepath.Join(dir, "ca.crt")) + if err != nil { + t.Fatalf("read ca.crt: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("ca.crt did not append to pool") + } + return &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }, + } +} + +// startTLSServer boots a full sqi-server with HTTPS, broker TLS, broker auth +// and the REST enrollment endpoint all enabled, against certificates in dir. +func startTLSServer(t *testing.T, dbPath, dir string) *testServer { + t.Helper() + certFile := filepath.Join(dir, "server.crt") + keyFile := filepath.Join(dir, "server.key") + + ts := startBrokerAuthServer(t, dbPath, func(cfg *server.Config) { + cfg.HTTPTLS = config.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + cfg.NATSTLS = config.NATSTLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + cfg.NATSAuthEnrollmentEndpointEnabled = true + }) + // startBrokerAuthServer stamps ts.Scheme/ts.Client from cfg.HTTPTLS, so + // every shared helper already speaks https here. + return ts +} + +func TestTLSEndToEnd(t *testing.T) { + dir := tlsMaterial(t) + dbPath := filepath.Join(t.TempDir(), "sqi.db") + joinToken := seedJoinToken(t, dbPath, "tls-e2e") + ts := startTLSServer(t, dbPath, dir) + + // The API really is HTTPS, verified against the farm CA. + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, + "https://"+ts.HTTPAddr+"/readyz", nil) + if err != nil { + t.Fatalf("build readyz request: %v", err) + } + resp, err := ts.Client.Do(req) + if err != nil { + t.Fatalf("GET /readyz over HTTPS: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("/readyz status = %d, want 200", resp.StatusCode) + } + + farmID, queueID := seedFarmAndQueue(t, ts) + caFile := filepath.Join(dir, "ca.crt") + startRealWorkerNoWait(t, ts, farmID, queueID, nil, []string{ + // Broker TLS: this URL overrides the harness default because exec + // resolves duplicate environment entries to the last one. + "SQI_WORKER_NATS_URL=nats://" + ts.NATSAddr, + "SQI_WORKER_NATS_TLS_CA_FILE=" + caFile, + // Enrollment over HTTPS — the bootstrap-critical half. + "SQI_WORKER_NATS_SERVER_URL=https://" + ts.HTTPAddr, + "SQI_WORKER_NATS_SERVER_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_JOIN_TOKEN=" + joinToken, + }, true) + + if id := findOnlineWorker(t, ts, farmID, 45*time.Second); id == "" { + t.Fatal("worker never came online over TLS: enrollment over HTTPS or the TLS broker connection failed") + } + + // Registering proves the transport works. Running a job proves the whole + // path does: submitted over HTTPS, assigned over the TLS broker, executed, + // and its status read back over HTTPS. + jobID := submitJob(t, ts, farmID, queueID) + if got := pollJobStatus(t, ts, jobID, []string{"completed", "failed", "canceled"}, 60*time.Second); got != "completed" { + t.Errorf("job %s ended %q, want completed", jobID, got) + } +} + +func TestTLSWorkerWithoutCAIsRefused(t *testing.T) { + dir := tlsMaterial(t) + dbPath := filepath.Join(t.TempDir(), "sqi.db") + joinToken := seedJoinToken(t, dbPath, "tls-nocert") + ts := startTLSServer(t, dbPath, dir) + + farmID, queueID := seedFarmAndQueue(t, ts) + // No CA configured anywhere: the worker can neither verify the server's + // certificate for enrollment nor speak TLS to the broker. + startRealWorkerNoWait(t, ts, farmID, queueID, nil, []string{ + "SQI_WORKER_NATS_URL=nats://" + ts.NATSAddr, + "SQI_WORKER_NATS_SERVER_URL=https://" + ts.HTTPAddr, + "SQI_WORKER_NATS_JOIN_TOKEN=" + joinToken, + }, true) + + if id := findOnlineWorker(t, ts, farmID, 10*time.Second); id != "" { + t.Fatalf("worker %s registered without any CA configured; TLS is not actually being enforced", id) + } +} + +// TestTLSMutualAuthEndToEnd covers the mTLS flow docs/tls.md promotes, with a +// real worker binary: `sqi-server tls init --client ` issues a client +// keypair, the worker presents it via nats.tls_cert_file/tls_key_file, and the +// broker accepts it. +// +// The unit tests in internal/bus hand-build the client tls.Config, so they +// never traverse natsclient.buildTLSOptions — which is the code an operator's +// configuration actually reaches, and where a wrong ExtKeyUsage on the +// generated certificate would bite. +func TestTLSMutualAuthEndToEnd(t *testing.T) { + dir, clientCert, clientKey := tlsMaterialWithClient(t, "render-01") + + dbPath := filepath.Join(t.TempDir(), "sqi.db") + joinToken := seedJoinToken(t, dbPath, "mtls-e2e") + + certFile := filepath.Join(dir, "server.crt") + keyFile := filepath.Join(dir, "server.key") + caFile := filepath.Join(dir, "ca.crt") + + ts := startBrokerAuthServer(t, dbPath, func(cfg *server.Config) { + cfg.HTTPTLS = config.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + cfg.NATSTLS = config.NATSTLSConfig{ + Enabled: true, CertFile: certFile, KeyFile: keyFile, + // Require a client certificate from every worker. + ClientCAFile: caFile, + } + cfg.NATSAuthEnrollmentEndpointEnabled = true + }) + + farmID, queueID := seedFarmAndQueue(t, ts) + startRealWorkerNoWait(t, ts, farmID, queueID, nil, []string{ + "SQI_WORKER_NATS_URL=nats://" + ts.NATSAddr, + "SQI_WORKER_NATS_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_TLS_CERT_FILE=" + clientCert, + "SQI_WORKER_NATS_TLS_KEY_FILE=" + clientKey, + "SQI_WORKER_NATS_SERVER_URL=https://" + ts.HTTPAddr, + "SQI_WORKER_NATS_SERVER_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_JOIN_TOKEN=" + joinToken, + }, true) + + if id := findOnlineWorker(t, ts, farmID, 45*time.Second); id == "" { + t.Fatal("worker never came online under mutual TLS: the generated client certificate was not accepted") + } +} + +// TestTLSMutualAuthRefusesWorkerWithoutCertificate is the negative half: the +// same broker must reject a worker that presents no client certificate. +// Without it, a broker that silently stopped requiring one would still pass +// the test above. +func TestTLSMutualAuthRefusesWorkerWithoutCertificate(t *testing.T) { + dir := tlsMaterial(t) + dbPath := filepath.Join(t.TempDir(), "sqi.db") + joinToken := seedJoinToken(t, dbPath, "mtls-negative") + + certFile := filepath.Join(dir, "server.crt") + keyFile := filepath.Join(dir, "server.key") + caFile := filepath.Join(dir, "ca.crt") + + ts := startBrokerAuthServer(t, dbPath, func(cfg *server.Config) { + cfg.HTTPTLS = config.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + cfg.NATSTLS = config.NATSTLSConfig{ + Enabled: true, CertFile: certFile, KeyFile: keyFile, ClientCAFile: caFile, + } + cfg.NATSAuthEnrollmentEndpointEnabled = true + }) + + farmID, queueID := seedFarmAndQueue(t, ts) + // Correct CA, correct join token, no client certificate. + startRealWorkerNoWait(t, ts, farmID, queueID, nil, []string{ + "SQI_WORKER_NATS_URL=nats://" + ts.NATSAddr, + "SQI_WORKER_NATS_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_SERVER_URL=https://" + ts.HTTPAddr, + "SQI_WORKER_NATS_SERVER_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_JOIN_TOKEN=" + joinToken, + }, true) + + if id := findOnlineWorker(t, ts, farmID, 10*time.Second); id != "" { + t.Fatalf("worker %s registered without a client certificate against an mTLS broker", id) + } +} + +// TestTLSMutualAuthWithIssuedCertificate covers the DAY-TWO path: a farm is +// already running mutual TLS, and a new worker is added with +// `sqi-server tls issue --client ` against the CA that already exists. +// +// `tls init` cannot do this — it refuses to overwrite the CA and fails as a +// whole, taking --client with it — so before `tls issue` existed there was no +// supported way to add a worker to an mTLS farm. The unit tests prove the +// certificate chains; this proves the broker actually accepts it. +func TestTLSMutualAuthWithIssuedCertificate(t *testing.T) { + // A farm that already exists, with no client certificate for our worker. + dir := tlsMaterial(t) + + // Day two: issue one from the SAME CA, exactly as the CLI does. + caPEM, err := os.ReadFile(filepath.Join(dir, "ca.crt")) + if err != nil { + t.Fatalf("read ca.crt: %v", err) + } + caKeyPEM, err := os.ReadFile(filepath.Join(dir, "ca.key")) + if err != nil { + t.Fatalf("read ca.key: %v", err) + } + ca, err := certgen.LoadCA(caPEM, caKeyPEM) + if err != nil { + t.Fatalf("LoadCA: %v", err) + } + leaf, err := ca.NewClientCert("render-day2", 365*24*time.Hour) + if err != nil { + t.Fatalf("NewClientCert: %v", err) + } + if err := certgen.WriteLeaf(dir, "client-render-day2", leaf); err != nil { + t.Fatalf("WriteLeaf: %v", err) + } + + dbPath := filepath.Join(t.TempDir(), "sqi.db") + joinToken := seedJoinToken(t, dbPath, "mtls-day2") + certFile := filepath.Join(dir, "server.crt") + keyFile := filepath.Join(dir, "server.key") + caFile := filepath.Join(dir, "ca.crt") + + ts := startBrokerAuthServer(t, dbPath, func(cfg *server.Config) { + cfg.HTTPTLS = config.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile} + cfg.NATSTLS = config.NATSTLSConfig{ + Enabled: true, CertFile: certFile, KeyFile: keyFile, ClientCAFile: caFile, + } + cfg.NATSAuthEnrollmentEndpointEnabled = true + }) + + farmID, queueID := seedFarmAndQueue(t, ts) + startRealWorkerNoWait(t, ts, farmID, queueID, nil, []string{ + "SQI_WORKER_NATS_URL=nats://" + ts.NATSAddr, + "SQI_WORKER_NATS_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_TLS_CERT_FILE=" + filepath.Join(dir, "client-render-day2.crt"), + "SQI_WORKER_NATS_TLS_KEY_FILE=" + filepath.Join(dir, "client-render-day2.key"), + "SQI_WORKER_NATS_SERVER_URL=https://" + ts.HTTPAddr, + "SQI_WORKER_NATS_SERVER_TLS_CA_FILE=" + caFile, + "SQI_WORKER_NATS_JOIN_TOKEN=" + joinToken, + }, true) + + if id := findOnlineWorker(t, ts, farmID, 45*time.Second); id == "" { + t.Fatal("a worker using a certificate issued from the existing CA was not accepted") + } +} diff --git a/test/integration/worker_binary_test.go b/test/integration/worker_binary_test.go index d0eadf75..783108a3 100644 --- a/test/integration/worker_binary_test.go +++ b/test/integration/worker_binary_test.go @@ -157,6 +157,23 @@ func startRealWorkerAnyOS(t *testing.T, ts *testServer, farmID, queueID string) // whether the POSIX-command skip applies; everything else is identical. func startRealWorkerCore(t *testing.T, ts *testServer, farmID, queueID string, extraArgs, extraEnv []string, allowWindows bool) string { t.Helper() + startRealWorkerNoWait(t, ts, farmID, queueID, extraArgs, extraEnv, allowWindows) + + // Poll GET /api/v1/workers until this worker appears with status "online". + // The worker needs to connect to NATS, register, and receive at least one + // heartbeat sweep before the server marks it online. + workerID := pollForOnlineWorker(t, ts, farmID, 30*time.Second) + t.Logf("real worker registered: id=%s", workerID) + return workerID +} + +// startRealWorkerNoWait starts a real worker subprocess and returns as soon as +// it is launched, without waiting for it to come online. +// +// A test asserting a worker must NEVER register needs this: for it, "never +// came online" is the expected outcome rather than a fatal error. +func startRealWorkerNoWait(t *testing.T, ts *testServer, farmID, queueID string, extraArgs, extraEnv []string, allowWindows bool) { + t.Helper() if runtime.GOOS == "windows" && !allowWindows { t.Skip("real-worker integration test requires a POSIX shell; skipping on Windows") @@ -221,19 +238,24 @@ func startRealWorkerCore(t *testing.T, ts *testServer, farmID, queueID string, e <-done // join the Wait goroutine; never call Wait again } }) - - // Poll GET /api/v1/workers until this worker appears with status "online". - // The worker needs to connect to NATS, register, and receive at least one - // heartbeat sweep before the server marks it online. - workerID := pollForOnlineWorker(t, ts, farmID, 30*time.Second) - t.Logf("real worker registered: id=%s", workerID) - return workerID } // pollForOnlineWorker polls GET /api/v1/workers until any worker in farmID // has status "online" and returns its ID, or fails the test after timeout. func pollForOnlineWorker(t *testing.T, ts *testServer, farmID string, timeout time.Duration) string { t.Helper() + id := findOnlineWorker(t, ts, farmID, timeout) + if id == "" { + t.Fatalf("pollForOnlineWorker: no online worker in farm %s within %s", farmID, timeout) + } + return id +} + +// findOnlineWorker is [pollForOnlineWorker] without the fatal: it returns "" +// on timeout. A test asserting a worker must NEVER register needs that, since +// for it a timeout is the expected result rather than a failure. +func findOnlineWorker(t *testing.T, ts *testServer, farmID string, timeout time.Duration) string { + t.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { @@ -248,13 +270,13 @@ func pollForOnlineWorker(t *testing.T, ts *testServer, farmID string, timeout ti req, err := http.NewRequestWithContext( context.Background(), http.MethodGet, - "http://"+ts.HTTPAddr+"/api/v1/workers", nil, + apiURL(ts, "/api/v1/workers"), nil, ) if err != nil { time.Sleep(200 * time.Millisecond) continue } - r, err := httpClient.Do(req) + r, err := clientFor(ts).Do(req) if err != nil { time.Sleep(200 * time.Millisecond) continue @@ -274,7 +296,6 @@ func pollForOnlineWorker(t *testing.T, ts *testServer, farmID string, timeout ti time.Sleep(200 * time.Millisecond) } - t.Fatalf("pollForOnlineWorker: no online worker in farm %s within %s", farmID, timeout) return "" }