[Metrics][Observability] Add basic Prometheus metrics to PEAS, including /healthz and /pprof

.gitignore

@@ -13,6 +13,7 @@
 
 # Don't commit binaries
 bin
+path-external-auth-server
 
 # Do not allow a multi-moduled projected
 go.work.sum

Makefile

@@ -2,10 +2,6 @@
 ### Makefile Helpers ###
 ########################
 
-.PHONY: list
-list: ## List all make targets
-	@${MAKE} -pRrn : -f $(MAKEFILE_LIST) 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | sort
-
 .PHONY: help
 .DEFAULT_GOAL := help
 help: ## Prints all the targets in all the Makefiles
@@ -139,3 +135,5 @@ get_portal_app_auth_status: ## Test auth/rate limit status for a Portal App ID (
 			localhost:10001 \
 			envoy.service.auth.v3.Authorization/Check | jq; \
 	fi
+
+include makefiles/local.mk

auth/auth_handler.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	envoy_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_auth "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3"
@@ -17,6 +18,7 @@ import (
 	"google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
 
+	"github.com/buildwithgrove/path-external-auth-server/metrics"
 	"github.com/buildwithgrove/path-external-auth-server/store"
 )
 
@@ -95,15 +97,31 @@ func (a *authHandler) Check(
 	ctx context.Context,
 	checkReq *envoy_auth.CheckRequest,
 ) (*envoy_auth.CheckResponse, error) {
+	startTime := time.Now()
+
 	// Get the HTTP request
 	req := checkReq.GetAttributes().GetRequest().GetHttp()
 	if req == nil {
+		metrics.RecordAuthRequest(
+			"", // portalAppID not available yet
+			"", // accountID not available yet
+			"error",
+			"invalid_request",
+			time.Since(startTime).Seconds(),
+		)
 		return getDeniedCheckResponse("HTTP request not found", envoy_type.StatusCode_BadRequest), nil
 	}
 
 	// Get the request path
 	path := req.GetPath()
 	if path == "" {
+		metrics.RecordAuthRequest(
+			"", // portalAppID not available yet
+			"", // accountID not available yet
+			"error",
+			"invalid_request",
+			time.Since(startTime).Seconds(),
+		)
 		return getDeniedCheckResponse("path not provided", envoy_type.StatusCode_BadRequest), nil
 	}
 
@@ -115,6 +133,13 @@ func (a *authHandler) Check(
 	portalAppID, err := extractPortalAppID(headers, path)
 	if err != nil {
 		a.logger.Debug().Err(err).Msg("🚫 unable to extract portal app ID from request")
+		metrics.RecordAuthRequest(
+			"", // portalAppID not available yet
+			"", // accountID not available yet
+			"error",
+			"invalid_request",
+			time.Since(startTime).Seconds(),
+		)
 		return getDeniedCheckResponse(err.Error(), envoy_type.StatusCode_BadRequest), nil
 	}
 	logger := a.logger.With("portal_app_id", portalAppID)
@@ -126,26 +151,56 @@ func (a *authHandler) Check(
 	portalApp, ok := a.getPortalApp(portalAppID)
 	if !ok {
 		logger.Debug().Msg("🚫 specified portal app not found: rejecting the request.")
+		metrics.RecordAuthRequest(
+			string(portalAppID),
+			"", // accountID not available yet
+			"denied",
+			"portal_app_not_found",
+			time.Since(startTime).Seconds(),
+		)
 		return getDeniedCheckResponse("portal app not found", envoy_type.StatusCode_NotFound), nil
 	}
 	logger = logger.With("account_id", portalApp.AccountID)
 
 	// Check if the Portal Application is authorized
 	if err := a.checkPortalAppAuthorized(headers, portalApp); err != nil {
 		logger.Debug().Err(err).Msg("🚫 request failed authorization: rejecting the request.")
+		metrics.RecordAuthRequest(
+			string(portalAppID),
+			string(portalApp.AccountID),
+			"denied",
+			"unauthorized",
+			time.Since(startTime).Seconds(),
+		)
 		return getDeniedCheckResponse(err.Error(), envoy_type.StatusCode_Unauthorized), nil
 	}
 
 	// Check if the Account is rate limited
 	if err := a.checkAccountRateLimited(portalApp); err != nil {
 		logger.Debug().Msg("🚫 account is rate limited: rejecting the request.")
+		metrics.RecordAuthRequest(
+			string(portalAppID),
+			string(portalApp.AccountID),
+			"denied",
+			"rate_limited",
+			time.Since(startTime).Seconds(),
+		)
 		return getDeniedCheckResponse(accountRateLimitMessage, envoy_type.StatusCode_TooManyRequests), nil
 	}
 
 	// Add Portal Application ID and Account ID to the headers
 	// to be passed upstream along the filter chain to the rate limiter.
 	httpHeaders := a.getHTTPHeaders(portalApp)
 
+	// Record successful authorization
+	metrics.RecordAuthRequest(
+		string(portalAppID),
+		string(portalApp.AccountID),
+		"authorized",
+		"",
+		time.Since(startTime).Seconds(),
+	)
+
 	// Return a valid response with the HTTP headers set
 	return getOKCheckResponse(httpHeaders), nil
 }
@@ -172,22 +227,35 @@ func (a *authHandler) getPortalApp(portalAppID store.PortalAppID) (*store.Portal
 //   - Returns nil if no authorization is required (Auth is nil or APIKey is empty)
 //   - Otherwise, performs API Key authorization
 func (a *authHandler) checkPortalAppAuthorized(headers http.Header, portalApp *store.PortalApp) error {
+	// If portal app does not require API key authorization, portalApp.Auth will be nil
+	// and no authorization will be performed by PEAS
 	if portalApp.Auth == nil || portalApp.Auth.APIKey == "" {
 		return nil
 	}
+
+	// Otherwise, perform API Key authorization
 	return a.apiKeyAuthorizer.authorizeRequest(headers, portalApp)
 }
 
 // checkAccountRateLimited checks if the account is rate limited.
 //   - Returns nil if the account is not eligible for rate limiting.
 //   - Returns an error if the account is rate limited.
 func (a *authHandler) checkAccountRateLimited(portalApp *store.PortalApp) error {
+	// If no rate limit is configured for this portal app, allow the request
 	if portalApp.RateLimit == nil {
+		metrics.RecordRateLimitCheck(string(portalApp.AccountID), "", "no_limit_configured")
 		return nil
 	}
+
+	// Check if the account has exceeded their rate limit
+	planType := string(portalApp.PlanType)
 	if a.rateLimitStore.IsAccountRateLimited(portalApp.AccountID) {
+		metrics.RecordRateLimitCheck(string(portalApp.AccountID), planType, "rate_limited")
 		return fmt.Errorf("account is rate limited")
 	}
+
+	// Account is within rate limits, allow the request
+	metrics.RecordRateLimitCheck(string(portalApp.AccountID), planType, "allowed")
 	return nil
 }
 

auth/auth_handler_test.go

@@ -55,10 +55,9 @@ func Test_Check(t *testing.T) {
 			mockPortalAppReturn: &store.PortalApp{
 				ID:        "portal_app_free",
 				AccountID: "account_1",
+				PlanType:  grovedb.PlanFree_DatabaseType,
 				Auth:      nil, // No auth required
-				RateLimit: &store.RateLimit{
-					PlanType: grovedb.PlanFree_DatabaseType,
-				},
+				RateLimit: &store.RateLimit{},
 			},
 		},
 		{
@@ -296,10 +295,9 @@ func Test_Check(t *testing.T) {
 			mockPortalAppReturn: &store.PortalApp{
 				ID:        "portal_app_rate_limited",
 				AccountID: "account_rate_limited",
+				PlanType:  grovedb.PlanFree_DatabaseType,
 				Auth:      nil,
-				RateLimit: &store.RateLimit{
-					PlanType: grovedb.PlanFree_DatabaseType,
-				},
+				RateLimit: &store.RateLimit{},
 			},
 		},
 		{
@@ -331,9 +329,9 @@ func Test_Check(t *testing.T) {
 			mockPortalAppReturn: &store.PortalApp{
 				ID:        "portal_app_unlimited_no_limit",
 				AccountID: "account_unlimited",
+				PlanType:  grovedb.PlanUnlimited_DatabaseType,
 				Auth:      nil,
 				RateLimit: &store.RateLimit{
-					PlanType:         grovedb.PlanUnlimited_DatabaseType,
 					MonthlyUserLimit: 0, // No specific limit
 				},
 			},

env.example

@@ -0,0 +1,47 @@
+# ================================
+# REQUIRED ENVIRONMENT VARIABLES
+# ================================
+
+# [REQUIRED]: PostgreSQL connection string for the PortalApp database used by the auth server.
+#   - Example: "postgresql://username:password@localhost:5432/dbname"
+POSTGRES_CONNECTION_STRING=
+
+# [REQUIRED]: GCP project ID for the data warehouse used by the rate limit store.
+#   - Example: "your-project-id"
+GCP_PROJECT_ID=
+
+# ================================
+# OPTIONAL ENVIRONMENT VARIABLES
+# ================================
+
+# [OPTIONAL]: Port to run the external auth server on.
+#   - Default: 10001 if not set
+PORT=10001
+
+# [OPTIONAL]: Port to run the Prometheus metrics server on.
+#   - Default: 9090 if not set
+METRICS_PORT=9090
+
+# [OPTIONAL]: Port to run the pprof server on.
+#   - Default: 6060 if not set
+PPROF_PORT=6060
+
+# [OPTIONAL]: Log level for the external auth server.
+#   - Default: "info" if not set
+#   - Options: "debug", "info", "warn", "error"
+LOGGER_LEVEL=info
+
+# [OPTIONAL]: Image tag/version for the application.
+#   - Default: "development" if not set
+#   - Used in /healthz endpoint response
+IMAGE_TAG=development
+
+# [OPTIONAL]: Refresh interval for the portal app store.
+#   - Default: 30s if not set
+#   - Examples: "30s", "1m", "2m30s"
+PORTAL_APP_STORE_REFRESH_INTERVAL=30s
+
+# [OPTIONAL]: Refresh interval for the rate limit store.
+#   - Default: 5m if not set
+#   - Examples: "30s", "1m", "2m30s"
+RATE_LIMIT_STORE_REFRESH_INTERVAL=5m