From 8eb7fbd94eb163edf5cbe47f809e60fb3bcac4f0 Mon Sep 17 00:00:00 2001 From: ssenerg Date: Tue, 17 Nov 2026 17:13:55 +0330 Subject: [PATCH 1/2] Wait for Scylla when the server starts When a machine boots, Docker starts the server and Scylla together, and Scylla takes a few seconds to accept connections. The server tried once, exited, and was restarted by Docker until Scylla was up: six times on the production server's last reboot. Now it tries again, with pauses growing from half a second to five, for up to two minutes, and migrate for up to one; both stop waiting as soon as they are stopped. Each attempt builds its cluster afresh, because gocql's token-aware host policy serves one session only. Co-Authored-By: Claude Opus 5.5 --- cmd/migrate/scylla.go | 32 ++++++++--- internal/app/server/run.go | 8 ++- internal/repos/scylla/repo.go | 5 +- internal/repos/scylla/session.go | 50 ++++++++++++++-- internal/repos/scylla/session_test.go | 82 +++++++++++++++++++++++++++ internal/repos/scylla/updates_test.go | 4 +- 6 files changed, 163 insertions(+), 18 deletions(-) create mode 100644 internal/repos/scylla/session_test.go diff --git a/cmd/migrate/scylla.go b/cmd/migrate/scylla.go index 5187548..5fa90b5 100644 --- a/cmd/migrate/scylla.go +++ b/cmd/migrate/scylla.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "time" "github.com/scylladb/gocqlx/v3/migrate" "github.com/spf13/cobra" @@ -40,6 +41,19 @@ func requirements(args []string) (*zap.Logger, *Config, error) { return logger, &cfg, nil } +// How long migrate waits for Scylla to accept connections, as it may still be +// starting. +const scyllaWait = time.Minute + +// commandContext is the command's context, which is nil when the command runs +// outside Execute. +func commandContext(cmd *cobra.Command) context.Context { + if ctx := cmd.Context(); ctx != nil { + return ctx + } + return context.Background() +} + func scyllaUp(cmd *cobra.Command, args []string) error { logger, cfg, err := requirements(args) if err != nil { @@ -47,11 +61,14 @@ func scyllaUp(cmd *cobra.Command, args []string) error { } defer func() { _ = logger.Sync() }() - if err := scylla.EnsureKeyspace(logger, cfg.Database.Scylla); err != nil { + ctx := commandContext(cmd) + connectCtx, cancel := context.WithTimeout(ctx, scyllaWait) + defer cancel() + if err := scylla.EnsureKeyspace(connectCtx, logger, cfg.Database.Scylla); err != nil { logger.Error("failed to prepare keyspace", zap.Error(err)) return err } - session, err := scylla.Open(logger, cfg.Database.Scylla) + session, err := scylla.Open(connectCtx, logger, cfg.Database.Scylla) if err != nil { return err } @@ -59,7 +76,7 @@ func scyllaUp(cmd *cobra.Command, args []string) error { logger.Info("applying migrations", zap.String("migration-dir", args[0])) migrate.DefaultAwaitSchemaAgreement = migrate.AwaitSchemaAgreementBeforeEachFile - if err := migrate.FromFS(cmd.Context(), session, os.DirFS(args[0])); err != nil { + if err := migrate.FromFS(ctx, session, os.DirFS(args[0])); err != nil { logger.Error("failed to apply migrations", zap.Error(err)) return fmt.Errorf("failed to apply migrations: %w", err) } @@ -74,16 +91,15 @@ func scyllaStatus(cmd *cobra.Command, args []string) error { } defer func() { _ = logger.Sync() }() - session, err := scylla.Open(logger, cfg.Database.Scylla) + ctx := commandContext(cmd) + connectCtx, cancel := context.WithTimeout(ctx, scyllaWait) + defer cancel() + session, err := scylla.Open(connectCtx, logger, cfg.Database.Scylla) if err != nil { return err } defer session.Close() - ctx := cmd.Context() - if ctx == nil { - ctx = context.Background() - } applied, err := migrate.List(ctx, session) if err != nil { return fmt.Errorf("failed to list migrations: %w", err) diff --git a/internal/app/server/run.go b/internal/app/server/run.go index de7dbf2..a4549b2 100644 --- a/internal/app/server/run.go +++ b/internal/app/server/run.go @@ -25,6 +25,10 @@ import ( const ( actorsStopTimeout = 15 * time.Second shutdownGrace = 15 * time.Second + // How long the server waits for Scylla to accept connections. When the + // machine boots, they start together, and exiting at once would only have + // Docker restart the server again and again until Scylla is up. + scyllaStartTimeout = 2 * time.Minute ) func Run(parent context.Context, cfg *Config) error { @@ -38,7 +42,9 @@ func Run(parent context.Context, cfg *Config) error { defer func() { _ = logger.Sync() }() mainLogger := logger.Named("main") - repo, err := scylla.NewRepository(logger.Named("scylla"), cfg.Database.Scylla) + scyllaCtx, scyllaCancel := context.WithTimeout(ctx, scyllaStartTimeout) + repo, err := scylla.NewRepository(scyllaCtx, logger.Named("scylla"), cfg.Database.Scylla) + scyllaCancel() if err != nil { return err } diff --git a/internal/repos/scylla/repo.go b/internal/repos/scylla/repo.go index b5a3567..0ea752a 100644 --- a/internal/repos/scylla/repo.go +++ b/internal/repos/scylla/repo.go @@ -21,8 +21,9 @@ type Repository struct { serial gocql.Consistency } -func NewRepository(logger *zap.Logger, cfg *Config) (*Repository, error) { - session, err := Open(logger, cfg) +// NewRepository connects to Scylla, waiting for it until ctx ends. +func NewRepository(ctx context.Context, logger *zap.Logger, cfg *Config) (*Repository, error) { + session, err := Open(ctx, logger, cfg) if err != nil { return nil, err } diff --git a/internal/repos/scylla/session.go b/internal/repos/scylla/session.go index 8dc997b..c2c29f1 100644 --- a/internal/repos/scylla/session.go +++ b/internal/repos/scylla/session.go @@ -1,8 +1,10 @@ package scylla import ( + "context" "errors" "fmt" + "time" "github.com/gocql/gocql" "github.com/scylladb/gocqlx/v3" @@ -68,9 +70,44 @@ func casUnknown(err error) bool { return errors.As(err, &cu) } -// Open connects to the configured keyspace. -func Open(logger *zap.Logger, cfg *Config) (gocqlx.Session, error) { - session, err := gocqlx.WrapSession(newCluster(cfg, cfg.Keyspace).CreateSession()) +// The pauses between attempts to connect while Scylla is not reachable yet: +// the first, doubling up to the last. Variables, so tests can shorten them. +var ( + firstRetryPause = 500 * time.Millisecond + maxRetryPause = 5 * time.Second +) + +// connect creates a session, trying again while Scylla is not reachable yet, +// until ctx ends: when a machine boots, the server and Scylla start together, +// and Scylla takes a while to accept connections. Each attempt gets a cluster +// of its own from cluster: gocql's host policies serve one session only. +func connect(ctx context.Context, logger *zap.Logger, cluster func() *gocql.ClusterConfig) (gocqlx.Session, error) { + pause := firstRetryPause + for attempt := 1; ; attempt++ { + session, err := gocqlx.WrapSession(cluster().CreateSession()) + if err == nil { + return session, nil + } + if ctx.Err() != nil { + return gocqlx.Session{}, err + } + logger.Warn("scylla is not reachable yet, trying again", + zap.Int("attempt", attempt), zap.Duration("in", pause), zap.Error(err)) + timer := time.NewTimer(pause) + select { + case <-ctx.Done(): + timer.Stop() + return gocqlx.Session{}, err + case <-timer.C: + } + pause = min(pause*2, maxRetryPause) + } +} + +// Open connects to the configured keyspace, waiting for Scylla until ctx +// ends. +func Open(ctx context.Context, logger *zap.Logger, cfg *Config) (gocqlx.Session, error) { + session, err := connect(ctx, logger, func() *gocql.ClusterConfig { return newCluster(cfg, cfg.Keyspace) }) if err != nil { logger.Error("failed to connect to scylla", zap.Error(err)) return gocqlx.Session{}, fmt.Errorf("failed to connect to scylla: %w", err) @@ -82,9 +119,10 @@ func Open(logger *zap.Logger, cfg *Config) (gocqlx.Session, error) { return session, nil } -// EnsureKeyspace creates the keyspace when it is missing. -func EnsureKeyspace(logger *zap.Logger, cfg *Config) error { - session, err := gocqlx.WrapSession(newCluster(cfg, "").CreateSession()) +// EnsureKeyspace creates the keyspace when it is missing, waiting for Scylla +// until ctx ends. +func EnsureKeyspace(ctx context.Context, logger *zap.Logger, cfg *Config) error { + session, err := connect(ctx, logger, func() *gocql.ClusterConfig { return newCluster(cfg, "") }) if err != nil { return fmt.Errorf("failed to connect to scylla: %w", err) } diff --git a/internal/repos/scylla/session_test.go b/internal/repos/scylla/session_test.go new file mode 100644 index 0000000..b600dbc --- /dev/null +++ b/internal/repos/scylla/session_test.go @@ -0,0 +1,82 @@ +package scylla + +import ( + "context" + "net" + "testing" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +// unreachable is a Config for an address nothing listens on, so that every +// attempt to connect fails at once. +func unreachable(t *testing.T) *Config { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + if err := ln.Close(); err != nil { + t.Fatal(err) + } + return &Config{ + Hosts: []string{addr}, Keyspace: "gochat", Consistency: "local_quorum", SerialConsistency: "local_serial", + Timeout: time.Second, ConnectTimeout: time.Second, NumConns: 1, DisableHostLookup: true, TLS: &TLSConfig{}, + } +} + +func shortPauses(t *testing.T) { + t.Helper() + first, last := firstRetryPause, maxRetryPause + firstRetryPause, maxRetryPause = 10*time.Millisecond, 40*time.Millisecond + t.Cleanup(func() { firstRetryPause, maxRetryPause = first, last }) +} + +func TestOpenWaitsForScyllaUntilTheContextEnds(t *testing.T) { + shortPauses(t) + core, logs := observer.New(zap.WarnLevel) + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + start := time.Now() + if _, err := Open(ctx, zap.New(core), unreachable(t)); err == nil { + t.Fatal("connected to an address nothing listens on") + } + if n := logs.FilterMessage("scylla is not reachable yet, trying again").Len(); n < 3 { + t.Fatalf("tried again %d times, want several", n) + } + if took := time.Since(start); took < 500*time.Millisecond || took > 5*time.Second { + t.Fatalf("gave up after %v, want soon after the context's 500ms", took) + } +} + +func TestOpenStopsWaitingWhenCanceled(t *testing.T) { + shortPauses(t) + core, logs := observer.New(zap.WarnLevel) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if _, err := Open(ctx, zap.New(core), unreachable(t)); err == nil { + t.Fatal("connected to an address nothing listens on") + } + if n := logs.FilterMessage("scylla is not reachable yet, trying again").Len(); n != 0 { + t.Fatalf("tried again %d times after the context was canceled", n) + } +} + +func TestEnsureKeyspaceWaitsForScylla(t *testing.T) { + shortPauses(t) + core, logs := observer.New(zap.WarnLevel) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + if err := EnsureKeyspace(ctx, zap.New(core), unreachable(t)); err == nil { + t.Fatal("connected to an address nothing listens on") + } + if n := logs.FilterMessage("scylla is not reachable yet, trying again").Len(); n < 2 { + t.Fatalf("tried again %d times, want several", n) + } +} diff --git a/internal/repos/scylla/updates_test.go b/internal/repos/scylla/updates_test.go index 381ab2c..9e749dd 100644 --- a/internal/repos/scylla/updates_test.go +++ b/internal/repos/scylla/updates_test.go @@ -20,7 +20,9 @@ func liveRepository(t *testing.T) *Repository { if host == "" { t.Skip("GOCHAT_TEST_SCYLLA is not set") } - r, err := NewRepository(zap.NewNop(), &Config{ + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + r, err := NewRepository(ctx, zap.NewNop(), &Config{ Hosts: []string{host}, Keyspace: "gochat", Consistency: "local_quorum", SerialConsistency: "local_serial", Timeout: 10 * time.Second, ConnectTimeout: 10 * time.Second, NumConns: 1, DisableHostLookup: true, TLS: &TLSConfig{}, }) From 24165803823abc9dae0c2ac832e4c1e610ee3c3e Mon Sep 17 00:00:00 2001 From: ssenerg Date: Tue, 17 Nov 2026 17:59:38 +0330 Subject: [PATCH 2/2] Build with Go 1.27.4, which fixes an HTTP server crash govulncheck fails CI on Go 1.27.3's GO-2026-6859: a race between http.MaxBytesReader and reading headers can crash an HTTP/1 server, and the web gateway, uploads and pushes all reach it. Go 1.27.4 fixes it. The builder image moves to golang:1.27.4-alpine3.24, pinned by its index digest like before. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- deployments/docker/Dockerfile | 2 +- deployments/docker/Dockerfile.template | 2 +- go.mod | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5ae43c6..359369d 100644 --- a/README.md +++ b/README.md @@ -495,7 +495,7 @@ that carry what people post. ## Requirements -- Go 1.27.3, [just](https://github.com/casey/just), [buf](https://buf.build) +- Go 1.27.4, [just](https://github.com/casey/just), [buf](https://buf.build) - Docker with Compose v2 ## Getting started diff --git a/deployments/docker/Dockerfile b/deployments/docker/Dockerfile index 4f94e24..6d59933 100644 --- a/deployments/docker/Dockerfile +++ b/deployments/docker/Dockerfile @@ -4,7 +4,7 @@ ARG DOCKER_REGISTRY=docker.io/ # Go cross-compiles: images for other platforms (--platform) build without # emulation. -FROM --platform=$BUILDPLATFORM ${DOCKER_REGISTRY}library/golang:1.27.3-alpine3.24@sha256:b8c9b82affbbb82a020bbcf998dad2ba9516944ac7e883524642a481a9505e9c AS builder +FROM --platform=$BUILDPLATFORM ${DOCKER_REGISTRY}library/golang:1.27.4-alpine3.24@sha256:70bd7ddea8b6cbe5d6943cde279d6b67feb10cc924fb9100d1d8757c1575b280 AS builder ARG TARGETOS TARGETARCH WORKDIR /src diff --git a/deployments/docker/Dockerfile.template b/deployments/docker/Dockerfile.template index 942c6b8..c9b2808 100644 --- a/deployments/docker/Dockerfile.template +++ b/deployments/docker/Dockerfile.template @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=docker.io/ # Go cross-compiles: images for other platforms (--platform) build without # emulation. -FROM --platform=$BUILDPLATFORM ${DOCKER_REGISTRY}library/golang:1.27.3-alpine3.24@sha256:b8c9b82affbbb82a020bbcf998dad2ba9516944ac7e883524642a481a9505e9c AS builder +FROM --platform=$BUILDPLATFORM ${DOCKER_REGISTRY}library/golang:1.27.4-alpine3.24@sha256:70bd7ddea8b6cbe5d6943cde279d6b67feb10cc924fb9100d1d8757c1575b280 AS builder ARG TARGETOS TARGETARCH WORKDIR /src diff --git a/go.mod b/go.mod index f43d2a8..f5878fa 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ssenerg/gochat -go 1.27.3 +go 1.27.4 require ( github.com/anthdm/hollywood v1.0.5