From 628553a54e91ac3af414f2276fef268afdd1b3d1 Mon Sep 17 00:00:00 2001 From: Gautam Kumar Date: Fri, 17 Apr 2026 14:59:07 +0530 Subject: [PATCH 1/6] feat(database): async connect with background retry and health middleware - Replace blocking Connect() with non-blocking background goroutine - Retry DB connection every 5s indefinitely; server starts without waiting - Add watchConnection() to detect and recover from lost connections - Expose IsHealthy() via atomic int32 flag (no mutex contention) - Add DBHealthCheck() middleware returning 503 when DB is unreachable - Add testing.go helper SetHealthForTest() for unit tests (non-production only) - Update main.go to use new Connect(cfg) signature and register middleware --- backend/cmd/server/main.go | 23 ++--- backend/internal/database/db.go | 113 +++++++++++++---------- backend/internal/database/testing.go | 13 +++ backend/internal/middleware/db_health.go | 22 +++++ 4 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 backend/internal/database/testing.go create mode 100644 backend/internal/middleware/db_health.go diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index ace7bce..070d7ec 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -1,35 +1,31 @@ package main import ( + "capuchin/internal/config" "capuchin/internal/database" "capuchin/internal/handlers" + "capuchin/internal/middleware" "capuchin/internal/routes" "capuchin/internal/services" "log" + "net/http" "time" "github.com/gin-gonic/gin" ) func main() { - // Bootstrapping schema at startup to keep local/dev deployments self-contained. - if err := database.Connect(); err != nil { - log.Fatalf("Startup failed: %v", err) - } - database.InitSchema() + database.Connect(config.Config) - // Periodic cleanup prevents the revoked-token table from growing forever. go func() { ticker := time.NewTicker(1 * time.Hour) - defer ticker.Stop() for range ticker.C { if err := database.CleanupTokens(); err != nil { - log.Printf("Error cleaning up expired tokens: %v", err) + log.Printf("token cleanup: error: %v", err) } } }() - // Handlers depend on interfaces so business logic can be swapped in tests. authService := services.NewAuthService() todoService := services.NewTodoService() @@ -38,21 +34,20 @@ func main() { r := gin.Default() - // Allow cross-origin requests so a separately hosted frontend can call this API. - // Restrict this in production to trusted origins. + // Restrict Access-Control-Allow-Origin to trusted origins in production. r.Use(func(c *gin.Context) { c.Writer.Header().Set("Access-Control-Allow-Origin", "*") c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE, PATCH") c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") if c.Request.Method == "OPTIONS" { - // Short-circuit preflight checks to avoid running downstream handlers. - c.AbortWithStatus(204) + c.AbortWithStatus(http.StatusNoContent) return } c.Next() }) - // Keep route wiring centralized so auth boundaries are easy to audit. + r.Use(middleware.DBHealthCheck()) + routes.SetupRoutes(r, authHandler, todoHandler) r.Run(":8080") diff --git a/backend/internal/database/db.go b/backend/internal/database/db.go index 10fd1e7..e1a801a 100644 --- a/backend/internal/database/db.go +++ b/backend/internal/database/db.go @@ -5,75 +5,92 @@ import ( "database/sql" "fmt" "log" + "sync/atomic" "time" _ "github.com/lib/pq" ) +// DB is the shared connection pool. Nil until the background goroutine +// successfully connects for the first time. var DB *sql.DB -const ( - dbMaxStartupAttempts = 5 - dbStartupBaseDelay = 2 * time.Second - dbStartupMaxDelay = 30 * time.Second -) +// dbHealthy is 1 when DB is reachable, 0 otherwise. +// Accessed exclusively via sync/atomic. +var dbHealthy int32 + +const retryInterval = 5 * time.Second -func Connect() error { - connStr := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=disable", - config.Config.POSTGRES_HOST, - config.Config.POSTGRES_USER, - config.Config.POSTGRES_PASSWORD, - config.Config.POSTGRES_DB, - config.Config.POSTGRES_PORT, +// Connect launches a background goroutine that attempts to open and ping +// Postgres on a fixed interval. It returns immediately without blocking the +// caller - the HTTP server starts before the DB is necessarily ready. +// The backend process never exits due to DB unavailability. +func Connect(cfg config.AppConfig) { + connStr := fmt.Sprintf( + "host=%s user=%s password=%s dbname=%s port=%d sslmode=disable", + cfg.POSTGRES_HOST, + cfg.POSTGRES_USER, + cfg.POSTGRES_PASSWORD, + cfg.POSTGRES_DB, + cfg.POSTGRES_PORT, ) - var err error - DB, err = sql.Open("postgres", connStr) - if err != nil { - return fmt.Errorf("failed to open database: %w", err) - } + go func() { + for { + db, err := sql.Open("postgres", connStr) + if err != nil { + log.Printf("database: connection open failed: %v - retry in %s", err, retryInterval) + atomic.StoreInt32(&dbHealthy, 0) + time.Sleep(retryInterval) + continue + } - // Conservative pool settings avoid exhausting DB connections in small deployments. - DB.SetMaxOpenConns(25) - DB.SetMaxIdleConns(5) - // Recycling connections helps recover from stale network state over long uptimes. - DB.SetConnMaxLifetime(5 * time.Minute) + if err := db.Ping(); err != nil { + log.Printf("database: ping failed: %v - retry in %s", err, retryInterval) + atomic.StoreInt32(&dbHealthy, 0) + _ = db.Close() + time.Sleep(retryInterval) + continue + } - if err = pingWithRetry(); err != nil { - DB.Close() - return fmt.Errorf("database unavailable after %d attempts: %w", dbMaxStartupAttempts, err) - } + db.SetMaxOpenConns(25) + db.SetMaxIdleConns(5) + db.SetConnMaxLifetime(5 * time.Minute) - log.Println("Database connection established") - return nil -} + DB = db + atomic.StoreInt32(&dbHealthy, 1) + log.Println("database: connection established") -func pingWithRetry() error { - delay := dbStartupBaseDelay - for attempt := 1; attempt <= dbMaxStartupAttempts; attempt++ { - if err := DB.Ping(); err == nil { - return nil - } else { - log.Printf("Database ping failed (attempt %d/%d): %v", attempt, dbMaxStartupAttempts, err) + watchConnection(db) } - if attempt < dbMaxStartupAttempts { - log.Printf("Retrying in %s...", delay) - time.Sleep(delay) - delay *= 2 - if delay > dbStartupMaxDelay { - delay = dbStartupMaxDelay - } + }() +} + +// watchConnection pings the DB on a fixed interval until the connection is lost. +func watchConnection(db *sql.DB) { + for { + time.Sleep(retryInterval) + if err := db.Ping(); err != nil { + log.Printf("database: connection lost: %v - reconnecting", err) + atomic.StoreInt32(&dbHealthy, 0) + _ = db.Close() + DB = nil + return } + atomic.StoreInt32(&dbHealthy, 1) } - return fmt.Errorf("database unreachable after %d attempts", dbMaxStartupAttempts) } -func InitSchema() { - log.Println("Database connection initialized. Assuming schema is already present.") +// IsHealthy reports whether the last DB ping succeeded. +func IsHealthy() bool { + return atomic.LoadInt32(&dbHealthy) == 1 } +// CleanupTokens deletes expired blacklisted tokens. func CleanupTokens() error { - // Expired tokens can be dropped because JWT expiration already invalidates them. - _, err := DB.Exec("DELETE FROM blacklisted_tokens WHERE expired_at < $1", time.Now()) + if DB == nil { + return fmt.Errorf("database: not connected") + } + _, err := DB.Exec("DELETE FROM blacklisted_tokens WHERE expired_at < NOW()") return err } diff --git a/backend/internal/database/testing.go b/backend/internal/database/testing.go new file mode 100644 index 0000000..95ae034 --- /dev/null +++ b/backend/internal/database/testing.go @@ -0,0 +1,13 @@ +//go:build !production + +package database + +// SetHealthForTest directly sets the DB health flag. +// Only compiled in non-production builds - use in tests only. +func SetHealthForTest(healthy bool) { + if healthy { + dbHealthy = 1 + } else { + dbHealthy = 0 + } +} diff --git a/backend/internal/middleware/db_health.go b/backend/internal/middleware/db_health.go new file mode 100644 index 0000000..f1ac7bf --- /dev/null +++ b/backend/internal/middleware/db_health.go @@ -0,0 +1,22 @@ +package middleware + +import ( + "capuchin/internal/database" + "net/http" + + "github.com/gin-gonic/gin" +) + +// DBHealthCheck returns 503 Service Unavailable when the database is unreachable. +// This prevents requests from reaching handlers that depend on a live DB connection. +func DBHealthCheck() gin.HandlerFunc { + return func(c *gin.Context) { + if !database.IsHealthy() { + c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{ + "error": "service temporarily unavailable", + }) + return + } + c.Next() + } +} From 39d63feca03bc1562a7a3cde8490db7de170c1c0 Mon Sep 17 00:00:00 2001 From: Gautam Kumar Date: Fri, 17 Apr 2026 15:07:34 +0530 Subject: [PATCH 2/6] remove .env file loading, use environment variables directly --- .gitignore | 1 + backend/go.mod | 1 - backend/go.sum | 2 -- backend/internal/config/config.go | 50 ------------------------------- 4 files changed, 1 insertion(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index de2736c..777f96f 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,4 @@ crash.*.log # personal docs/ideas.md backup/ +.kiro \ No newline at end of file diff --git a/backend/go.mod b/backend/go.mod index 0a580fc..963f5b7 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -6,7 +6,6 @@ require ( github.com/gin-gonic/gin v1.11.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 - github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.11.2 golang.org/x/crypto v0.48.0 ) diff --git a/backend/go.sum b/backend/go.sum index cd59fd3..1b82300 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -34,8 +34,6 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index bb2537a..31eb087 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -1,15 +1,11 @@ package config import ( - "errors" "fmt" "log" "os" - "path/filepath" "strconv" "strings" - - "github.com/joho/godotenv" ) type AppConfig struct { @@ -24,8 +20,6 @@ var Config AppConfig var JWTKey []byte func init() { - loadEnvFile() - postgresPort, err := postgresPortFromEnv() if err != nil { log.Fatal(err) @@ -53,50 +47,6 @@ func init() { log.Println("Configuration loaded successfully.") } -func loadEnvFile() { - cwd, err := os.Getwd() - if err != nil { - log.Fatalf("failed to determine current working directory: %v", err) - } - - envPath, err := findEnvFile(cwd) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - log.Println("No .env file found in current or parent directories; using existing environment variables.") - return - } - log.Fatalf("failed to locate .env file: %v", err) - } - - if err := godotenv.Load(envPath); err != nil { - log.Fatalf("failed to load .env file %q: %v", envPath, err) - } - - log.Printf("Loaded environment variables from %s", envPath) -} - -func findEnvFile(startDir string) (string, error) { - dir := startDir - for { - candidate := filepath.Join(dir, ".env") - info, err := os.Stat(candidate) - if err == nil && !info.IsDir() { - return candidate, nil - } - if err != nil && !errors.Is(err, os.ErrNotExist) { - return "", err - } - - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent - } - - return "", os.ErrNotExist -} - func postgresPortFromEnv() (int, error) { rawPort := os.Getenv("POSTGRES_PORT") if rawPort == "" { From dbf5abfe8975baba740c974ffe507d79ec4fc876 Mon Sep 17 00:00:00 2001 From: Gautam Kumar Date: Fri, 17 Apr 2026 15:33:50 +0530 Subject: [PATCH 3/6] refactor(database): replace polling watchConnection with atomic.Pointer and sql.DB pool management --- backend/internal/database/db.go | 74 ++++++++++------------- backend/internal/middleware/auth.go | 2 +- backend/internal/services/auth_service.go | 6 +- backend/internal/services/todo_service.go | 10 +-- 4 files changed, 40 insertions(+), 52 deletions(-) diff --git a/backend/internal/database/db.go b/backend/internal/database/db.go index e1a801a..51f99d1 100644 --- a/backend/internal/database/db.go +++ b/backend/internal/database/db.go @@ -11,20 +11,29 @@ import ( _ "github.com/lib/pq" ) -// DB is the shared connection pool. Nil until the background goroutine -// successfully connects for the first time. -var DB *sql.DB +// db is the shared connection pool. Access via GetDB(). +// Uses atomic.Pointer to avoid data races on connect/reconnect. +var db atomic.Pointer[sql.DB] -// dbHealthy is 1 when DB is reachable, 0 otherwise. -// Accessed exclusively via sync/atomic. +// dbHealthy is 1 when the DB connection is established, 0 otherwise. var dbHealthy int32 const retryInterval = 5 * time.Second -// Connect launches a background goroutine that attempts to open and ping -// Postgres on a fixed interval. It returns immediately without blocking the -// caller - the HTTP server starts before the DB is necessarily ready. -// The backend process never exits due to DB unavailability. +// GetDB returns the active connection pool, or nil if not yet connected. +func GetDB() *sql.DB { + return db.Load() +} + +// IsHealthy reports whether the DB is currently reachable. +func IsHealthy() bool { + return atomic.LoadInt32(&dbHealthy) == 1 +} + +// Connect launches a background goroutine that establishes the DB connection +// and retries on failure. Returns immediately — the server starts without +// waiting for the DB. Once connected, sql.DB manages the pool internally; +// no polling loop is needed. func Connect(cfg config.AppConfig) { connStr := fmt.Sprintf( "host=%s user=%s password=%s dbname=%s port=%d sslmode=disable", @@ -37,60 +46,39 @@ func Connect(cfg config.AppConfig) { go func() { for { - db, err := sql.Open("postgres", connStr) + conn, err := sql.Open("postgres", connStr) if err != nil { - log.Printf("database: connection open failed: %v - retry in %s", err, retryInterval) - atomic.StoreInt32(&dbHealthy, 0) + log.Printf("database: open failed: %v — retry in %s", err, retryInterval) time.Sleep(retryInterval) continue } - if err := db.Ping(); err != nil { - log.Printf("database: ping failed: %v - retry in %s", err, retryInterval) + if err := conn.Ping(); err != nil { + log.Printf("database: ping failed: %v — retry in %s", err, retryInterval) + _ = conn.Close() atomic.StoreInt32(&dbHealthy, 0) - _ = db.Close() time.Sleep(retryInterval) continue } - db.SetMaxOpenConns(25) - db.SetMaxIdleConns(5) - db.SetConnMaxLifetime(5 * time.Minute) + conn.SetMaxOpenConns(25) + conn.SetMaxIdleConns(5) + conn.SetConnMaxLifetime(5 * time.Minute) - DB = db + db.Store(conn) atomic.StoreInt32(&dbHealthy, 1) log.Println("database: connection established") - - watchConnection(db) - } - }() -} - -// watchConnection pings the DB on a fixed interval until the connection is lost. -func watchConnection(db *sql.DB) { - for { - time.Sleep(retryInterval) - if err := db.Ping(); err != nil { - log.Printf("database: connection lost: %v - reconnecting", err) - atomic.StoreInt32(&dbHealthy, 0) - _ = db.Close() - DB = nil return } - atomic.StoreInt32(&dbHealthy, 1) - } -} - -// IsHealthy reports whether the last DB ping succeeded. -func IsHealthy() bool { - return atomic.LoadInt32(&dbHealthy) == 1 + }() } // CleanupTokens deletes expired blacklisted tokens. func CleanupTokens() error { - if DB == nil { + conn := GetDB() + if conn == nil { return fmt.Errorf("database: not connected") } - _, err := DB.Exec("DELETE FROM blacklisted_tokens WHERE expired_at < NOW()") + _, err := conn.Exec("DELETE FROM blacklisted_tokens WHERE expired_at < NOW()") return err } diff --git a/backend/internal/middleware/auth.go b/backend/internal/middleware/auth.go index 10f9adf..751ecc5 100644 --- a/backend/internal/middleware/auth.go +++ b/backend/internal/middleware/auth.go @@ -24,7 +24,7 @@ func AuthRequired() gin.HandlerFunc { var exists bool // Check revocation before claim extraction so logout takes effect immediately. - err := database.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM blacklisted_tokens WHERE token=$1)", tokenStr).Scan(&exists) + err := database.GetDB().QueryRow("SELECT EXISTS(SELECT 1 FROM blacklisted_tokens WHERE token=$1)", tokenStr).Scan(&exists) if err != nil && err != sql.ErrNoRows { c.AbortWithStatusJSON(503, gin.H{"error": "authentication service unavailable"}) return diff --git a/backend/internal/services/auth_service.go b/backend/internal/services/auth_service.go index 7b6da99..471f47a 100644 --- a/backend/internal/services/auth_service.go +++ b/backend/internal/services/auth_service.go @@ -44,7 +44,7 @@ func (s *authService) Signup(email, password string) (*models.User, error) { PasswordHash: string(hash), } - _, err = database.DB.Exec("INSERT INTO users (id, email, password_hash) VALUES ($1, $2, $3)", u.ID, u.Email, u.PasswordHash) + _, err = database.GetDB().Exec("INSERT INTO users (id, email, password_hash) VALUES ($1, $2, $3)", u.ID, u.Email, u.PasswordHash) if err != nil { errStr := err.Error() // Convert storage-specific duplicate key errors into a stable domain error for handlers. @@ -59,7 +59,7 @@ func (s *authService) Signup(email, password string) (*models.User, error) { func (s *authService) Login(email, password string) (string, error) { var u models.User - err := database.DB.QueryRow("SELECT id, email, password_hash FROM users WHERE email=$1", email).Scan(&u.ID, &u.Email, &u.PasswordHash) + err := database.GetDB().QueryRow("SELECT id, email, password_hash FROM users WHERE email=$1", email).Scan(&u.ID, &u.Email, &u.PasswordHash) if err != nil { // Use one response for unknown user and wrong password to avoid account enumeration. return "", ErrInvalidCredentials @@ -116,7 +116,7 @@ func (s *authService) Logout(tokenStr string) error { } // Idempotent logout avoids surfacing harmless duplicate requests as server errors. - _, err := database.DB.Exec("INSERT INTO blacklisted_tokens (token, expired_at) VALUES ($1, $2) ON CONFLICT (token) DO NOTHING", tokenStr, expTime) + _, err := database.GetDB().Exec("INSERT INTO blacklisted_tokens (token, expired_at) VALUES ($1, $2) ON CONFLICT (token) DO NOTHING", tokenStr, expTime) if err != nil { return ErrDatabase } diff --git a/backend/internal/services/todo_service.go b/backend/internal/services/todo_service.go index df36148..c94375d 100644 --- a/backend/internal/services/todo_service.go +++ b/backend/internal/services/todo_service.go @@ -28,7 +28,7 @@ func NewTodoService() TodoService { func (s *todoService) GetTodos(userID uuid.UUID) ([]models.Todo, error) { // Scope every read by user_id so one user can never read another user's todos. - rows, err := database.DB.Query("SELECT id, item, completed FROM todos WHERE user_id=$1", userID) + rows, err := database.GetDB().Query("SELECT id, item, completed FROM todos WHERE user_id=$1", userID) if err != nil { return nil, ErrDatabase } @@ -59,7 +59,7 @@ func (s *todoService) AddTodo(userID uuid.UUID, item string, completed bool) (*m Completed: completed, } - _, err := database.DB.Exec("INSERT INTO todos (id, item, completed, user_id) VALUES ($1, $2, $3, $4)", t.ID, t.Item, t.Completed, t.UserID) + _, err := database.GetDB().Exec("INSERT INTO todos (id, item, completed, user_id) VALUES ($1, $2, $3, $4)", t.ID, t.Item, t.Completed, t.UserID) if err != nil { return nil, ErrDatabase } @@ -70,7 +70,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet if item == nil && completed == nil { // Empty PATCH requests are treated as a read to keep the endpoint idempotent. var t models.Todo - err := database.DB.QueryRow("SELECT id, item, completed FROM todos WHERE id=$1 AND user_id=$2", todoID, userID).Scan(&t.ID, &t.Item, &t.Completed) + err := database.GetDB().QueryRow("SELECT id, item, completed FROM todos WHERE id=$1 AND user_id=$2", todoID, userID).Scan(&t.ID, &t.Item, &t.Completed) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTodoNotFound @@ -81,7 +81,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet } var t models.Todo - err := database.DB.QueryRow(` + err := database.GetDB().QueryRow(` UPDATE todos -- COALESCE preserves existing values when fields are omitted from PATCH payloads. SET item = COALESCE($1, item), @@ -99,7 +99,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet } func (s *todoService) DeleteTodo(userID, todoID uuid.UUID) error { - res, err := database.DB.Exec("DELETE FROM todos WHERE id=$1 AND user_id=$2", todoID, userID) + res, err := database.GetDB().Exec("DELETE FROM todos WHERE id=$1 AND user_id=$2", todoID, userID) if err != nil { return ErrDatabase } From 7e6434c4c777fcb2e5ad1df670aaf4b5be5617c2 Mon Sep 17 00:00:00 2001 From: Gautam Kumar Date: Fri, 17 Apr 2026 15:49:14 +0530 Subject: [PATCH 4/6] feat(observability): add structured JSON logger and DB health monitor --- backend/cmd/server/main.go | 8 ++- backend/internal/database/db.go | 65 ++++++++++++++++------- backend/internal/database/testing.go | 13 ----- backend/internal/logger/logger.go | 61 +++++++++++++++++++++ backend/internal/middleware/db_health.go | 22 -------- backend/internal/routes/routes.go | 17 ++++++ backend/internal/services/auth_service.go | 3 ++ backend/internal/services/todo_service.go | 7 +++ 8 files changed, 138 insertions(+), 58 deletions(-) delete mode 100644 backend/internal/database/testing.go create mode 100644 backend/internal/logger/logger.go delete mode 100644 backend/internal/middleware/db_health.go diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 070d7ec..751427e 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -4,10 +4,9 @@ import ( "capuchin/internal/config" "capuchin/internal/database" "capuchin/internal/handlers" - "capuchin/internal/middleware" + "capuchin/internal/logger" "capuchin/internal/routes" "capuchin/internal/services" - "log" "net/http" "time" @@ -16,12 +15,13 @@ import ( func main() { database.Connect(config.Config) + database.StartHealthMonitor() go func() { ticker := time.NewTicker(1 * time.Hour) for range ticker.C { if err := database.CleanupTokens(); err != nil { - log.Printf("token cleanup: error: %v", err) + logger.Error("token.cleanup", "failed to delete expired tokens", err) } } }() @@ -46,8 +46,6 @@ func main() { c.Next() }) - r.Use(middleware.DBHealthCheck()) - routes.SetupRoutes(r, authHandler, todoHandler) r.Run(":8080") diff --git a/backend/internal/database/db.go b/backend/internal/database/db.go index 51f99d1..f0c5285 100644 --- a/backend/internal/database/db.go +++ b/backend/internal/database/db.go @@ -2,9 +2,10 @@ package database import ( "capuchin/internal/config" + "capuchin/internal/logger" + "context" "database/sql" "fmt" - "log" "sync/atomic" "time" @@ -15,25 +16,20 @@ import ( // Uses atomic.Pointer to avoid data races on connect/reconnect. var db atomic.Pointer[sql.DB] -// dbHealthy is 1 when the DB connection is established, 0 otherwise. -var dbHealthy int32 - -const retryInterval = 5 * time.Second +const ( + retryInterval = 5 * time.Second + monitorInterval = 30 * time.Second + pingTimeout = 2 * time.Second +) // GetDB returns the active connection pool, or nil if not yet connected. func GetDB() *sql.DB { return db.Load() } -// IsHealthy reports whether the DB is currently reachable. -func IsHealthy() bool { - return atomic.LoadInt32(&dbHealthy) == 1 -} - // Connect launches a background goroutine that establishes the DB connection // and retries on failure. Returns immediately — the server starts without -// waiting for the DB. Once connected, sql.DB manages the pool internally; -// no polling loop is needed. +// waiting for the DB. Once connected, sql.DB manages the pool internally. func Connect(cfg config.AppConfig) { connStr := fmt.Sprintf( "host=%s user=%s password=%s dbname=%s port=%d sslmode=disable", @@ -48,15 +44,18 @@ func Connect(cfg config.AppConfig) { for { conn, err := sql.Open("postgres", connStr) if err != nil { - log.Printf("database: open failed: %v — retry in %s", err, retryInterval) + logger.Error("database", "open failed", err) time.Sleep(retryInterval) continue } - if err := conn.Ping(); err != nil { - log.Printf("database: ping failed: %v — retry in %s", err, retryInterval) + ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) + err = conn.PingContext(ctx) + cancel() + + if err != nil { + logger.Error("database", "ping failed during connect", err) _ = conn.Close() - atomic.StoreInt32(&dbHealthy, 0) time.Sleep(retryInterval) continue } @@ -66,13 +65,43 @@ func Connect(cfg config.AppConfig) { conn.SetConnMaxLifetime(5 * time.Minute) db.Store(conn) - atomic.StoreInt32(&dbHealthy, 1) - log.Println("database: connection established") + logger.Info("database", "connection established") return } }() } +// StartHealthMonitor pings the DB every 30s and logs a structured warning +// when unreachable. Intended for observability only — does not gate requests. +// Call after Connect(). +func StartHealthMonitor() { + go func() { + ticker := time.NewTicker(monitorInterval) + defer ticker.Stop() + for range ticker.C { + conn := GetDB() + if conn == nil { + logger.Warn("database.monitor", "DB not yet connected", nil) + continue + } + + ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) + err := conn.PingContext(ctx) + cancel() + + if err != nil { + logger.Warn("database.monitor", "DB unreachable", err) + } else { + stats := conn.Stats() + logger.Info("database.monitor", fmt.Sprintf( + "healthy — open=%d idle=%d waitCount=%d", + stats.OpenConnections, stats.Idle, stats.WaitCount, + )) + } + } + }() +} + // CleanupTokens deletes expired blacklisted tokens. func CleanupTokens() error { conn := GetDB() diff --git a/backend/internal/database/testing.go b/backend/internal/database/testing.go deleted file mode 100644 index 95ae034..0000000 --- a/backend/internal/database/testing.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build !production - -package database - -// SetHealthForTest directly sets the DB health flag. -// Only compiled in non-production builds - use in tests only. -func SetHealthForTest(healthy bool) { - if healthy { - dbHealthy = 1 - } else { - dbHealthy = 0 - } -} diff --git a/backend/internal/logger/logger.go b/backend/internal/logger/logger.go new file mode 100644 index 0000000..2f5e87e --- /dev/null +++ b/backend/internal/logger/logger.go @@ -0,0 +1,61 @@ +// Package logger provides structured JSON logging with component tagging. +// Log lines are machine-parseable so monitoring tools can filter by component, +// level, or error type without regex scraping. +package logger + +import ( + "encoding/json" + "log" + "os" + "time" +) + +type Level string + +const ( + LevelInfo Level = "INFO" + LevelWarn Level = "WARN" + LevelError Level = "ERROR" +) + +type entry struct { + Time string `json:"time"` + Level Level `json:"level"` + Component string `json:"component"` + Message string `json:"msg"` + Error string `json:"error,omitempty"` +} + +var out = log.New(os.Stdout, "", 0) + +func write(level Level, component, msg, errStr string) { + e := entry{ + Time: time.Now().UTC().Format(time.RFC3339), + Level: level, + Component: component, + Message: msg, + Error: errStr, + } + b, _ := json.Marshal(e) + out.Println(string(b)) +} + +func Info(component, msg string) { + write(LevelInfo, component, msg, "") +} + +func Warn(component, msg string, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + write(LevelWarn, component, msg, errStr) +} + +func Error(component, msg string, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + write(LevelError, component, msg, errStr) +} diff --git a/backend/internal/middleware/db_health.go b/backend/internal/middleware/db_health.go deleted file mode 100644 index f1ac7bf..0000000 --- a/backend/internal/middleware/db_health.go +++ /dev/null @@ -1,22 +0,0 @@ -package middleware - -import ( - "capuchin/internal/database" - "net/http" - - "github.com/gin-gonic/gin" -) - -// DBHealthCheck returns 503 Service Unavailable when the database is unreachable. -// This prevents requests from reaching handlers that depend on a live DB connection. -func DBHealthCheck() gin.HandlerFunc { - return func(c *gin.Context) { - if !database.IsHealthy() { - c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{ - "error": "service temporarily unavailable", - }) - return - } - c.Next() - } -} diff --git a/backend/internal/routes/routes.go b/backend/internal/routes/routes.go index dd77457..209cde1 100644 --- a/backend/internal/routes/routes.go +++ b/backend/internal/routes/routes.go @@ -1,8 +1,11 @@ package routes import ( + "capuchin/internal/database" "capuchin/internal/handlers" "capuchin/internal/middleware" + "context" + "time" "github.com/gin-gonic/gin" ) @@ -12,6 +15,20 @@ func SetupRoutes(router *gin.Engine, authHandler *handlers.AuthHandler, todoHand router.Use(middleware.ErrorHandler()) router.GET("/health", func(c *gin.Context) { + db := database.GetDB() + if db == nil { + c.JSON(503, gin.H{"status": "unavailable", "reason": "database not connected"}) + return + } + + ctx, cancel := context.WithTimeout(c.Request.Context(), 2*time.Second) + defer cancel() + + if err := db.PingContext(ctx); err != nil { + c.JSON(503, gin.H{"status": "unavailable", "reason": "database unreachable"}) + return + } + c.JSON(200, gin.H{"status": "ok"}) }) diff --git a/backend/internal/services/auth_service.go b/backend/internal/services/auth_service.go index 471f47a..1f8b8fb 100644 --- a/backend/internal/services/auth_service.go +++ b/backend/internal/services/auth_service.go @@ -3,6 +3,7 @@ package services import ( "capuchin/internal/config" "capuchin/internal/database" + "capuchin/internal/logger" "capuchin/internal/models" "errors" "strings" @@ -51,6 +52,7 @@ func (s *authService) Signup(email, password string) (*models.User, error) { if strings.Contains(errStr, "unique constraint") || strings.Contains(errStr, "duplicate key value") { return nil, ErrUserExists } + logger.Error("auth.signup", "insert user failed", err) return nil, ErrDatabase } @@ -118,6 +120,7 @@ func (s *authService) Logout(tokenStr string) error { // Idempotent logout avoids surfacing harmless duplicate requests as server errors. _, err := database.GetDB().Exec("INSERT INTO blacklisted_tokens (token, expired_at) VALUES ($1, $2) ON CONFLICT (token) DO NOTHING", tokenStr, expTime) if err != nil { + logger.Error("auth.logout", "insert blacklisted token failed", err) return ErrDatabase } diff --git a/backend/internal/services/todo_service.go b/backend/internal/services/todo_service.go index c94375d..4099af2 100644 --- a/backend/internal/services/todo_service.go +++ b/backend/internal/services/todo_service.go @@ -2,6 +2,7 @@ package services import ( "capuchin/internal/database" + "capuchin/internal/logger" "capuchin/internal/models" "database/sql" "errors" @@ -30,6 +31,7 @@ func (s *todoService) GetTodos(userID uuid.UUID) ([]models.Todo, error) { // Scope every read by user_id so one user can never read another user's todos. rows, err := database.GetDB().Query("SELECT id, item, completed FROM todos WHERE user_id=$1", userID) if err != nil { + logger.Error("todo.get", "query failed", err) return nil, ErrDatabase } defer rows.Close() @@ -45,6 +47,7 @@ func (s *todoService) GetTodos(userID uuid.UUID) ([]models.Todo, error) { } if err := rows.Err(); err != nil { + logger.Error("todo.get", "rows iteration failed", err) return nil, ErrDatabase } @@ -61,6 +64,7 @@ func (s *todoService) AddTodo(userID uuid.UUID, item string, completed bool) (*m _, err := database.GetDB().Exec("INSERT INTO todos (id, item, completed, user_id) VALUES ($1, $2, $3, $4)", t.ID, t.Item, t.Completed, t.UserID) if err != nil { + logger.Error("todo.add", "insert failed", err) return nil, ErrDatabase } return t, nil @@ -75,6 +79,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet if errors.Is(err, sql.ErrNoRows) { return nil, ErrTodoNotFound } + logger.Error("todo.update", "read-only fetch failed", err) return nil, ErrDatabase } return &t, nil @@ -93,6 +98,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet if errors.Is(err, sql.ErrNoRows) { return nil, ErrTodoNotFound } + logger.Error("todo.update", "update query failed", err) return nil, ErrDatabase } return &t, nil @@ -101,6 +107,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet func (s *todoService) DeleteTodo(userID, todoID uuid.UUID) error { res, err := database.GetDB().Exec("DELETE FROM todos WHERE id=$1 AND user_id=$2", todoID, userID) if err != nil { + logger.Error("todo.delete", "delete query failed", err) return ErrDatabase } From 3f1d46e41ff9cdccff530f349f37f51173e8d2e5 Mon Sep 17 00:00:00 2001 From: Gautam Kumar Date: Fri, 17 Apr 2026 16:17:15 +0530 Subject: [PATCH 5/6] feat(database): two-speed health monitor with degraded mode signaling --- backend/internal/database/db.go | 113 +++++++++++++++++----- backend/internal/database/errors.go | 13 +++ backend/internal/middleware/auth.go | 1 + backend/internal/routes/routes.go | 16 +-- backend/internal/services/auth_service.go | 2 + backend/internal/services/todo_service.go | 6 ++ 6 files changed, 111 insertions(+), 40 deletions(-) create mode 100644 backend/internal/database/errors.go diff --git a/backend/internal/database/db.go b/backend/internal/database/db.go index f0c5285..e29e1a9 100644 --- a/backend/internal/database/db.go +++ b/backend/internal/database/db.go @@ -16,10 +16,18 @@ import ( // Uses atomic.Pointer to avoid data races on connect/reconnect. var db atomic.Pointer[sql.DB] +// dbHealthy is 1 when the last monitor ping succeeded, 0 otherwise. +var dbHealthy atomic.Int32 + +// degraded is a channel used to signal the monitor to switch to fast polling. +// Buffered so callers never block. +var degraded = make(chan struct{}, 1) + const ( - retryInterval = 5 * time.Second - monitorInterval = 30 * time.Second - pingTimeout = 2 * time.Second + retryInterval = 5 * time.Second + healthyInterval = 30 * time.Second + degradedInterval = 5 * time.Second + pingTimeout = 2 * time.Second ) // GetDB returns the active connection pool, or nil if not yet connected. @@ -27,9 +35,23 @@ func GetDB() *sql.DB { return db.Load() } +// IsHealthy reports the cached DB health state set by StartHealthMonitor. +func IsHealthy() bool { + return dbHealthy.Load() == 1 +} + +// MarkDegraded signals the monitor to switch to fast polling immediately. +// Safe to call from any goroutine; never blocks. +func MarkDegraded() { + select { + case degraded <- struct{}{}: + default: // already signalled, drop + } +} + // Connect launches a background goroutine that establishes the DB connection // and retries on failure. Returns immediately — the server starts without -// waiting for the DB. Once connected, sql.DB manages the pool internally. +// waiting for the DB. func Connect(cfg config.AppConfig) { connStr := fmt.Sprintf( "host=%s user=%s password=%s dbname=%s port=%d sslmode=disable", @@ -65,43 +87,82 @@ func Connect(cfg config.AppConfig) { conn.SetConnMaxLifetime(5 * time.Minute) db.Store(conn) + dbHealthy.Store(1) logger.Info("database", "connection established") return } }() } -// StartHealthMonitor pings the DB every 30s and logs a structured warning -// when unreachable. Intended for observability only — does not gate requests. -// Call after Connect(). +// StartHealthMonitor runs a two-speed ping loop: +// - healthy: pings every 30s for observability +// - degraded: pings every 5s to detect recovery as soon as possible +// +// Switches to degraded mode when a ping fails or MarkDegraded() is called +// (e.g. from a service that hit a query error). Backs off to healthy interval +// once a ping succeeds again. func StartHealthMonitor() { go func() { - ticker := time.NewTicker(monitorInterval) - defer ticker.Stop() - for range ticker.C { - conn := GetDB() - if conn == nil { - logger.Warn("database.monitor", "DB not yet connected", nil) - continue - } + interval := healthyInterval + timer := time.NewTimer(interval) + defer timer.Stop() - ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) - err := conn.PingContext(ctx) - cancel() + for { + select { + case <-degraded: + // A query error was reported — switch to fast polling immediately + // without waiting for the current timer to fire. + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + interval = degradedInterval + timer.Reset(interval) - if err != nil { - logger.Warn("database.monitor", "DB unreachable", err) - } else { - stats := conn.Stats() - logger.Info("database.monitor", fmt.Sprintf( - "healthy — open=%d idle=%d waitCount=%d", - stats.OpenConnections, stats.Idle, stats.WaitCount, - )) + case <-timer.C: + ping(interval == degradedInterval) + if IsHealthy() { + interval = healthyInterval + } else { + interval = degradedInterval + } + timer.Reset(interval) } } }() } +func ping(wasDegraded bool) { + conn := GetDB() + if conn == nil { + dbHealthy.Store(0) + logger.Warn("database.monitor", "DB not yet connected", nil) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) + err := conn.PingContext(ctx) + cancel() + + if err != nil { + dbHealthy.Store(0) + logger.Warn("database.monitor", "DB unreachable", err) + return + } + + if wasDegraded { + logger.Info("database.monitor", "DB recovered") + } + dbHealthy.Store(1) + stats := conn.Stats() + logger.Info("database.monitor", fmt.Sprintf( + "healthy — open=%d idle=%d waitCount=%d", + stats.OpenConnections, stats.Idle, stats.WaitCount, + )) +} + // CleanupTokens deletes expired blacklisted tokens. func CleanupTokens() error { conn := GetDB() diff --git a/backend/internal/database/errors.go b/backend/internal/database/errors.go new file mode 100644 index 0000000..9615a0a --- /dev/null +++ b/backend/internal/database/errors.go @@ -0,0 +1,13 @@ +package database + +import "database/sql" + +// HandleQueryError signals the health monitor to switch to fast polling +// when a real connectivity error occurs, as opposed to expected errors +// like sql.ErrNoRows which don't indicate DB unavailability. +func HandleQueryError(err error) { + if err == nil || err == sql.ErrNoRows { + return + } + MarkDegraded() +} diff --git a/backend/internal/middleware/auth.go b/backend/internal/middleware/auth.go index 751ecc5..55f3adf 100644 --- a/backend/internal/middleware/auth.go +++ b/backend/internal/middleware/auth.go @@ -26,6 +26,7 @@ func AuthRequired() gin.HandlerFunc { // Check revocation before claim extraction so logout takes effect immediately. err := database.GetDB().QueryRow("SELECT EXISTS(SELECT 1 FROM blacklisted_tokens WHERE token=$1)", tokenStr).Scan(&exists) if err != nil && err != sql.ErrNoRows { + database.HandleQueryError(err) c.AbortWithStatusJSON(503, gin.H{"error": "authentication service unavailable"}) return } diff --git a/backend/internal/routes/routes.go b/backend/internal/routes/routes.go index 209cde1..d29c6a8 100644 --- a/backend/internal/routes/routes.go +++ b/backend/internal/routes/routes.go @@ -4,8 +4,6 @@ import ( "capuchin/internal/database" "capuchin/internal/handlers" "capuchin/internal/middleware" - "context" - "time" "github.com/gin-gonic/gin" ) @@ -15,20 +13,10 @@ func SetupRoutes(router *gin.Engine, authHandler *handlers.AuthHandler, todoHand router.Use(middleware.ErrorHandler()) router.GET("/health", func(c *gin.Context) { - db := database.GetDB() - if db == nil { - c.JSON(503, gin.H{"status": "unavailable", "reason": "database not connected"}) + if !database.IsHealthy() { + c.JSON(503, gin.H{"status": "unavailable"}) return } - - ctx, cancel := context.WithTimeout(c.Request.Context(), 2*time.Second) - defer cancel() - - if err := db.PingContext(ctx); err != nil { - c.JSON(503, gin.H{"status": "unavailable", "reason": "database unreachable"}) - return - } - c.JSON(200, gin.H{"status": "ok"}) }) diff --git a/backend/internal/services/auth_service.go b/backend/internal/services/auth_service.go index 1f8b8fb..2c6dd86 100644 --- a/backend/internal/services/auth_service.go +++ b/backend/internal/services/auth_service.go @@ -53,6 +53,7 @@ func (s *authService) Signup(email, password string) (*models.User, error) { return nil, ErrUserExists } logger.Error("auth.signup", "insert user failed", err) + database.HandleQueryError(err) return nil, ErrDatabase } @@ -121,6 +122,7 @@ func (s *authService) Logout(tokenStr string) error { _, err := database.GetDB().Exec("INSERT INTO blacklisted_tokens (token, expired_at) VALUES ($1, $2) ON CONFLICT (token) DO NOTHING", tokenStr, expTime) if err != nil { logger.Error("auth.logout", "insert blacklisted token failed", err) + database.HandleQueryError(err) return ErrDatabase } diff --git a/backend/internal/services/todo_service.go b/backend/internal/services/todo_service.go index 4099af2..d09e867 100644 --- a/backend/internal/services/todo_service.go +++ b/backend/internal/services/todo_service.go @@ -32,6 +32,7 @@ func (s *todoService) GetTodos(userID uuid.UUID) ([]models.Todo, error) { rows, err := database.GetDB().Query("SELECT id, item, completed FROM todos WHERE user_id=$1", userID) if err != nil { logger.Error("todo.get", "query failed", err) + database.HandleQueryError(err) return nil, ErrDatabase } defer rows.Close() @@ -48,6 +49,7 @@ func (s *todoService) GetTodos(userID uuid.UUID) ([]models.Todo, error) { if err := rows.Err(); err != nil { logger.Error("todo.get", "rows iteration failed", err) + database.HandleQueryError(err) return nil, ErrDatabase } @@ -65,6 +67,7 @@ func (s *todoService) AddTodo(userID uuid.UUID, item string, completed bool) (*m _, err := database.GetDB().Exec("INSERT INTO todos (id, item, completed, user_id) VALUES ($1, $2, $3, $4)", t.ID, t.Item, t.Completed, t.UserID) if err != nil { logger.Error("todo.add", "insert failed", err) + database.HandleQueryError(err) return nil, ErrDatabase } return t, nil @@ -80,6 +83,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet return nil, ErrTodoNotFound } logger.Error("todo.update", "read-only fetch failed", err) + database.HandleQueryError(err) return nil, ErrDatabase } return &t, nil @@ -99,6 +103,7 @@ func (s *todoService) UpdateTodo(userID, todoID uuid.UUID, item *string, complet return nil, ErrTodoNotFound } logger.Error("todo.update", "update query failed", err) + database.HandleQueryError(err) return nil, ErrDatabase } return &t, nil @@ -108,6 +113,7 @@ func (s *todoService) DeleteTodo(userID, todoID uuid.UUID) error { res, err := database.GetDB().Exec("DELETE FROM todos WHERE id=$1 AND user_id=$2", todoID, userID) if err != nil { logger.Error("todo.delete", "delete query failed", err) + database.HandleQueryError(err) return ErrDatabase } From d3044f8f9458e76262a7cd77e2c368e6c8c4ed69 Mon Sep 17 00:00:00 2001 From: Gautam Kumar Date: Fri, 17 Apr 2026 16:26:53 +0530 Subject: [PATCH 6/6] refactor(database): fix sql.Open allocation, rename constants, explicit degraded state, reduce log noise --- backend/internal/database/db.go | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/backend/internal/database/db.go b/backend/internal/database/db.go index e29e1a9..d6ee3f4 100644 --- a/backend/internal/database/db.go +++ b/backend/internal/database/db.go @@ -24,10 +24,10 @@ var dbHealthy atomic.Int32 var degraded = make(chan struct{}, 1) const ( - retryInterval = 5 * time.Second - healthyInterval = 30 * time.Second - degradedInterval = 5 * time.Second - pingTimeout = 2 * time.Second + connectRetryInterval = 5 * time.Second + healthyPingInterval = 30 * time.Second + degradedPingInterval = 5 * time.Second + pingTimeout = 2 * time.Second ) // GetDB returns the active connection pool, or nil if not yet connected. @@ -49,9 +49,8 @@ func MarkDegraded() { } } -// Connect launches a background goroutine that establishes the DB connection -// and retries on failure. Returns immediately — the server starts without -// waiting for the DB. +// Connect opens the connection pool once and launches a background goroutine +// that pings until ready, retrying on failure. Returns immediately. func Connect(cfg config.AppConfig) { connStr := fmt.Sprintf( "host=%s user=%s password=%s dbname=%s port=%d sslmode=disable", @@ -62,30 +61,29 @@ func Connect(cfg config.AppConfig) { cfg.POSTGRES_PORT, ) + // sql.Open only validates the DSN — allocate the pool once outside the retry loop. + conn, err := sql.Open("postgres", connStr) + if err != nil { + logger.Error("database", "failed to open connection pool", err) + return + } + + conn.SetMaxOpenConns(25) + conn.SetMaxIdleConns(5) + conn.SetConnMaxLifetime(5 * time.Minute) + go func() { for { - conn, err := sql.Open("postgres", connStr) - if err != nil { - logger.Error("database", "open failed", err) - time.Sleep(retryInterval) - continue - } - ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) - err = conn.PingContext(ctx) + err := conn.PingContext(ctx) cancel() if err != nil { - logger.Error("database", "ping failed during connect", err) - _ = conn.Close() - time.Sleep(retryInterval) + logger.Error("database", "ping failed, retrying", err) + time.Sleep(connectRetryInterval) continue } - conn.SetMaxOpenConns(25) - conn.SetMaxIdleConns(5) - conn.SetConnMaxLifetime(5 * time.Minute) - db.Store(conn) dbHealthy.Store(1) logger.Info("database", "connection established") @@ -98,13 +96,12 @@ func Connect(cfg config.AppConfig) { // - healthy: pings every 30s for observability // - degraded: pings every 5s to detect recovery as soon as possible // -// Switches to degraded mode when a ping fails or MarkDegraded() is called -// (e.g. from a service that hit a query error). Backs off to healthy interval -// once a ping succeeds again. +// Switches to degraded mode when a ping fails or MarkDegraded() is called. +// Backs off to healthy interval once a ping succeeds again. func StartHealthMonitor() { go func() { - interval := healthyInterval - timer := time.NewTimer(interval) + isDegraded := false + timer := time.NewTimer(healthyPingInterval) defer timer.Stop() for { @@ -112,29 +109,32 @@ func StartHealthMonitor() { case <-degraded: // A query error was reported — switch to fast polling immediately // without waiting for the current timer to fire. - if !timer.Stop() { - select { - case <-timer.C: - default: + if !isDegraded { + if !timer.Stop() { + select { + case <-timer.C: + default: + } } + isDegraded = true + timer.Reset(degradedPingInterval) } - interval = degradedInterval - timer.Reset(interval) case <-timer.C: - ping(interval == degradedInterval) + pingMonitor(isDegraded) if IsHealthy() { - interval = healthyInterval + isDegraded = false + timer.Reset(healthyPingInterval) } else { - interval = degradedInterval + isDegraded = true + timer.Reset(degradedPingInterval) } - timer.Reset(interval) } } }() } -func ping(wasDegraded bool) { +func pingMonitor(isDegraded bool) { conn := GetDB() if conn == nil { dbHealthy.Store(0) @@ -152,15 +152,15 @@ func ping(wasDegraded bool) { return } - if wasDegraded { - logger.Info("database.monitor", "DB recovered") + if isDegraded { + // Only log recovery and stats when coming back from a degraded state. + stats := conn.Stats() + logger.Info("database.monitor", fmt.Sprintf( + "DB recovered — open=%d idle=%d waitCount=%d", + stats.OpenConnections, stats.Idle, stats.WaitCount, + )) } dbHealthy.Store(1) - stats := conn.Stats() - logger.Info("database.monitor", fmt.Sprintf( - "healthy — open=%d idle=%d waitCount=%d", - stats.OpenConnections, stats.Idle, stats.WaitCount, - )) } // CleanupTokens deletes expired blacklisted tokens.