From 8a9f531585c0b3fef2a09ad507b55bb70d237688 Mon Sep 17 00:00:00 2001 From: Gray Fay Date: Fri, 11 Sep 2026 00:56:00 -0400 Subject: [PATCH] feat: tell callers which states actually have data Street-level data is Ohio only. A caller querying anywhere else gets an empty result set, which is indistinguishable from a malformed query or a broken service -- so they open a ticket instead of reading a number. GET /api/v1/coverage summary by state GET /api/v1/coverage?state=OH per-county breakdown Derived from ohio_addresses, not the datasets table. datasets tracks uploads rather than contents, and the original Ohio import did not come through the uploader, so it would report zero for the only state that has data. Cached for ten minutes rather than aggregated per request -- that GROUP BY over every address row on every call is the same mistake the address search count query was making. Coverage only changes when a dataset is imported, so the import invalidates the snapshot and the TTL is just a backstop. A failed rebuild serves the previous snapshot instead of a 500: this is descriptive metadata, and last week's answer beats an error. Coverage needs a valid API key but no permission scope. Gating it would 403 every key already issued -- none carry a "coverage" permission -- and the endpoint exists so a caller can find out what to ask for before asking. It returns no address, ZIP or boundary content. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01S2nhUJ4Zdsxf7cP9DapsY3 --- handlers/coverage_handlers.go | 54 +++++++++ main.go | 5 + middleware/auth.go | 3 + services/auth_service.go | 9 ++ services/coverage_integration_test.go | 112 ++++++++++++++++++ services/coverage_service.go | 162 ++++++++++++++++++++++++++ services/dataset_service.go | 4 + 7 files changed, 349 insertions(+) create mode 100644 handlers/coverage_handlers.go create mode 100644 services/coverage_integration_test.go create mode 100644 services/coverage_service.go diff --git a/handlers/coverage_handlers.go b/handlers/coverage_handlers.go new file mode 100644 index 0000000..7853f12 --- /dev/null +++ b/handlers/coverage_handlers.go @@ -0,0 +1,54 @@ +package handlers + +import ( + "net/http" + "strings" + + "geocoding-api/services" + + "github.com/labstack/echo/v4" +) + +// GetCoverageHandler reports which states and counties have address data. +// +// Street-level data is currently Ohio only. A caller querying anywhere else +// gets an empty result set, which looks exactly like a malformed query or a +// broken service -- so they open a ticket. This makes that self-service. +// +// GET /api/v1/coverage summary by state +// GET /api/v1/coverage?state=OH per-county breakdown for one state +func GetCoverageHandler(c echo.Context) error { + db := services.GetDB() + + if state := strings.TrimSpace(c.QueryParam("state")); state != "" { + counties, err := services.GetStateCoverage(db, state) + if err != nil { + return c.JSON(http.StatusInternalServerError, GeocodeResponse{ + Success: false, + Error: "Failed to read coverage", + }) + } + + return c.JSON(http.StatusOK, GeocodeResponse{ + Success: true, + Data: map[string]interface{}{ + "state": strings.ToUpper(state), + "counties": counties, + "count": len(counties), + }, + }) + } + + snapshot, err := services.GetCoverage(db) + if err != nil { + return c.JSON(http.StatusInternalServerError, GeocodeResponse{ + Success: false, + Error: "Failed to read coverage", + }) + } + + return c.JSON(http.StatusOK, GeocodeResponse{ + Success: true, + Data: snapshot, + }) +} diff --git a/main.go b/main.go index 3979b7c..45fdea7 100644 --- a/main.go +++ b/main.go @@ -296,6 +296,11 @@ func main() { protected.Use(middleware.APIKeyAuth()) protected.Use(middleware.UsageHeader()) + // What data the service actually holds. Answering this without it means + // querying a state and inferring from an empty result, which is + // indistinguishable from a broken query. + protected.GET("/coverage", handlers.GetCoverageHandler) + // Geocoding endpoints protected.GET("/geocode/:zipcode", handlers.GetZipCodeHandler) protected.GET("/search", handlers.SearchZipCodesHandler) diff --git a/middleware/auth.go b/middleware/auth.go index 83a2b95..f0ac12c 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -199,6 +199,9 @@ func APIKeyAuth() echo.MiddlewareFunc { // getEndpointName extracts the endpoint name from the path for categorization func getEndpointName(path string) string { + if strings.Contains(path, "/coverage") { + return "coverage" + } if strings.Contains(path, "/geocode/") { return "geocode" } diff --git a/services/auth_service.go b/services/auth_service.go index 4316d9d..c1da7dd 100644 --- a/services/auth_service.go +++ b/services/auth_service.go @@ -1261,6 +1261,15 @@ func (as *AuthService) HasPermission(apiKey *models.APIKey, endpoint string) boo "admin": "admin", } + // Coverage is service metadata -- which states hold data -- and returns no + // address, ZIP or boundary content. Gating it on a scope would 403 every + // key already issued, since none of them carry a "coverage" permission, + // and the endpoint exists precisely so a caller can find out what to ask + // for before asking. A valid key is still required. + if endpoint == "coverage" { + return true + } + requiredPermission, exists := permissionMap[endpoint] if !exists { return false // Unknown endpoint diff --git a/services/coverage_integration_test.go b/services/coverage_integration_test.go new file mode 100644 index 0000000..9453355 --- /dev/null +++ b/services/coverage_integration_test.go @@ -0,0 +1,112 @@ +package services + +import ( + "testing" + "time" +) + +// The endpoint exists so a caller can find out that, say, Michigan has no +// street data -- instead of querying it, getting nothing, and filing a ticket. +func TestCoverageReportsWhatIsLoaded(t *testing.T) { + db := setupCountTestDB(t) + ResetCoverageCache() + + snapshot, err := GetCoverage(db) + if err != nil { + t.Fatalf("coverage: %v", err) + } + if len(snapshot.States) == 0 { + t.Fatal("no states reported for a seeded table") + } + + var total int + for _, s := range snapshot.States { + if s.State == "" { + t.Error("a state row came back with no state code") + } + if s.Counties <= 0 { + t.Errorf("state %s reports %d counties", s.State, s.Counties) + } + total += s.Addresses + } + if total != snapshot.TotalRows { + t.Errorf("per-state addresses sum to %d, total says %d", total, snapshot.TotalRows) + } + t.Logf("coverage: %d state(s), %d addresses", len(snapshot.States), snapshot.TotalRows) +} + +// The alternative to caching is a GROUP BY over every address row on every +// request, which is the mistake the address search count query was making. +func TestCoverageIsCachedAndInvalidatedOnImport(t *testing.T) { + db := setupCountTestDB(t) + ResetCoverageCache() + + first, err := GetCoverage(db) + if err != nil { + t.Fatalf("first: %v", err) + } + + second, err := GetCoverage(db) + if err != nil { + t.Fatalf("second: %v", err) + } + if !first.GeneratedAt.Equal(second.GeneratedAt) { + t.Error("second call recomputed instead of serving the cached snapshot") + } + + // An import is the only thing that changes coverage. + time.Sleep(2 * time.Millisecond) + ResetCoverageCache() + third, err := GetCoverage(db) + if err != nil { + t.Fatalf("third: %v", err) + } + if !third.GeneratedAt.After(second.GeneratedAt) { + t.Error("snapshot was not rebuilt after the cache was invalidated") + } +} + +// The drill-down has to agree with the summary, or the two views contradict +// each other and neither can be trusted. +func TestStateCoverageAgreesWithTheSummary(t *testing.T) { + db := setupCountTestDB(t) + ResetCoverageCache() + + snapshot, err := GetCoverage(db) + if err != nil { + t.Fatalf("coverage: %v", err) + } + + for _, s := range snapshot.States { + counties, err := GetStateCoverage(db, s.State) + if err != nil { + t.Fatalf("state coverage for %s: %v", s.State, err) + } + if len(counties) != s.Counties { + t.Errorf("state %s: summary says %d counties, drill-down returns %d", + s.State, s.Counties, len(counties)) + } + + var sum int + for _, c := range counties { + sum += c.Addresses + } + if sum != s.Addresses { + t.Errorf("state %s: counties sum to %d, summary says %d", s.State, sum, s.Addresses) + } + } +} + +// An unknown state is an empty list, not an error -- that is the answer the +// caller came for. +func TestUnknownStateReturnsEmptyNotError(t *testing.T) { + db := setupCountTestDB(t) + + counties, err := GetStateCoverage(db, "MI") + if err != nil { + t.Fatalf("unknown state should not error: %v", err) + } + if len(counties) != 0 { + t.Errorf("got %d counties for an unloaded state", len(counties)) + } +} diff --git a/services/coverage_service.go b/services/coverage_service.go new file mode 100644 index 0000000..0f759d6 --- /dev/null +++ b/services/coverage_service.go @@ -0,0 +1,162 @@ +package services + +import ( + "database/sql" + "sync" + "time" + + "geocoding-api/database" +) + +// coverageTTL is how long a computed snapshot is served before it is rebuilt. +// +// Coverage only changes when a dataset is imported, which is a deliberate +// admin action measured in weeks, so a stale answer is never far wrong. The +// TTL exists because the alternative is a GROUP BY over every address row on +// every request -- the same mistake the count query in address search was +// making. +const coverageTTL = 10 * time.Minute + +// StateCoverage is what the service holds for one state. +type StateCoverage struct { + State string `json:"state"` + Counties int `json:"counties"` + Addresses int `json:"addresses"` +} + +// CountyCoverage is the per-county breakdown within a state. +type CountyCoverage struct { + County string `json:"county"` + Addresses int `json:"addresses"` +} + +// Coverage answers "what data do you actually have". +// +// Without it a caller querying a state that was never loaded gets an empty +// result set, which is indistinguishable from a bad query or a broken service. +// That is a support ticket every time. The datasets table is admin-only and +// tracks uploads rather than contents, so it cannot answer this: the original +// Ohio import did not come through the uploader and would be missing entirely. +type Coverage struct { + States []StateCoverage `json:"states"` + TotalRows int `json:"total_addresses"` + GeneratedAt time.Time `json:"generated_at"` + MaxAgeSecs int `json:"max_age_seconds"` +} + +type coverageCache struct { + mu sync.Mutex + snapshot *Coverage + builtAt time.Time +} + +var coverage = &coverageCache{} + +// GetCoverage returns the current snapshot, recomputing it when stale. +// +// The lock is held across the query on purpose. A thundering herd of requests +// arriving on a cold cache would otherwise each run the aggregate; holding it +// means the first one pays and the rest wait for that result. +func GetCoverage(db *sql.DB) (*Coverage, error) { + coverage.mu.Lock() + defer coverage.mu.Unlock() + + if coverage.snapshot != nil && time.Since(coverage.builtAt) < coverageTTL { + return coverage.snapshot, nil + } + + snapshot, err := buildCoverage(db) + if err != nil { + // Serve a stale snapshot rather than an error. Coverage is + // descriptive metadata; last week's answer is far more useful than a + // 500, and it is never badly wrong. + if coverage.snapshot != nil { + return coverage.snapshot, nil + } + return nil, err + } + + coverage.snapshot = snapshot + coverage.builtAt = time.Now() + return snapshot, nil +} + +func buildCoverage(db *sql.DB) (*Coverage, error) { + if db == nil { + db = database.DB + } + + rows, err := db.Query(` + SELECT COALESCE(NULLIF(region, ''), 'OH') AS state, + COUNT(DISTINCT county) AS counties, + COUNT(*) AS addresses + FROM ohio_addresses + GROUP BY COALESCE(NULLIF(region, ''), 'OH') + ORDER BY COUNT(*) DESC + `) + if err != nil { + return nil, err + } + defer rows.Close() + + snapshot := &Coverage{ + States: []StateCoverage{}, + GeneratedAt: time.Now(), + MaxAgeSecs: int(coverageTTL.Seconds()), + } + + for rows.Next() { + var sc StateCoverage + if err := rows.Scan(&sc.State, &sc.Counties, &sc.Addresses); err != nil { + return nil, err + } + snapshot.TotalRows += sc.Addresses + snapshot.States = append(snapshot.States, sc) + } + if err := rows.Err(); err != nil { + return nil, err + } + + return snapshot, nil +} + +// GetStateCoverage returns the per-county breakdown for one state. +// +// Not cached: it is the drill-down, asked for far less often than the summary, +// and it is already narrowed by state. +func GetStateCoverage(db *sql.DB, state string) ([]CountyCoverage, error) { + if db == nil { + db = database.DB + } + + rows, err := db.Query(` + SELECT county, COUNT(*) AS addresses + FROM ohio_addresses + WHERE COALESCE(NULLIF(region, ''), 'OH') = UPPER($1) + AND county <> '' + GROUP BY county + ORDER BY county + `, state) + if err != nil { + return nil, err + } + defer rows.Close() + + counties := []CountyCoverage{} + for rows.Next() { + var cc CountyCoverage + if err := rows.Scan(&cc.County, &cc.Addresses); err != nil { + return nil, err + } + counties = append(counties, cc) + } + return counties, rows.Err() +} + +// ResetCoverageCache drops the cached snapshot. Called after an import so the +// next request reflects newly loaded data rather than waiting out the TTL. +func ResetCoverageCache() { + coverage.mu.Lock() + defer coverage.mu.Unlock() + coverage.snapshot = nil +} diff --git a/services/dataset_service.go b/services/dataset_service.go index fc5ef87..8408fd5 100644 --- a/services/dataset_service.go +++ b/services/dataset_service.go @@ -426,6 +426,10 @@ func (s *DatasetService) ProcessGeoJSONDataset(datasetID int) error { dataset.FilePath, importer.failed) } + // An import is the only thing that changes coverage, so drop the cached + // snapshot instead of serving a stale one for the rest of the TTL. + ResetCoverageCache() + log.Printf("Successfully processed dataset %d: %d records imported, %d duplicates skipped, %d failed", datasetID, recordCount, skippedDuplicates, importer.failed) return nil