diff --git a/handlers/coverage_handlers.go b/handlers/coverage_handlers.go new file mode 100644 index 0000000..7853f12 --- /dev/null +++ b/handlers/coverage_handlers.go @@ -0,0 +1,54 @@ +package handlers + +import ( + "net/http" + "strings" + + "geocoding-api/services" + + "github.com/labstack/echo/v4" +) + +// GetCoverageHandler reports which states and counties have address data. +// +// Street-level data is currently Ohio only. A caller querying anywhere else +// gets an empty result set, which looks exactly like a malformed query or a +// broken service -- so they open a ticket. This makes that self-service. +// +// GET /api/v1/coverage summary by state +// GET /api/v1/coverage?state=OH per-county breakdown for one state +func GetCoverageHandler(c echo.Context) error { + db := services.GetDB() + + if state := strings.TrimSpace(c.QueryParam("state")); state != "" { + counties, err := services.GetStateCoverage(db, state) + if err != nil { + return c.JSON(http.StatusInternalServerError, GeocodeResponse{ + Success: false, + Error: "Failed to read coverage", + }) + } + + return c.JSON(http.StatusOK, GeocodeResponse{ + Success: true, + Data: map[string]interface{}{ + "state": strings.ToUpper(state), + "counties": counties, + "count": len(counties), + }, + }) + } + + snapshot, err := services.GetCoverage(db) + if err != nil { + return c.JSON(http.StatusInternalServerError, GeocodeResponse{ + Success: false, + Error: "Failed to read coverage", + }) + } + + return c.JSON(http.StatusOK, GeocodeResponse{ + Success: true, + Data: snapshot, + }) +} diff --git a/main.go b/main.go index 3979b7c..45fdea7 100644 --- a/main.go +++ b/main.go @@ -296,6 +296,11 @@ func main() { protected.Use(middleware.APIKeyAuth()) protected.Use(middleware.UsageHeader()) + // What data the service actually holds. Answering this without it means + // querying a state and inferring from an empty result, which is + // indistinguishable from a broken query. + protected.GET("/coverage", handlers.GetCoverageHandler) + // Geocoding endpoints protected.GET("/geocode/:zipcode", handlers.GetZipCodeHandler) protected.GET("/search", handlers.SearchZipCodesHandler) diff --git a/middleware/auth.go b/middleware/auth.go index 83a2b95..f0ac12c 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -199,6 +199,9 @@ func APIKeyAuth() echo.MiddlewareFunc { // getEndpointName extracts the endpoint name from the path for categorization func getEndpointName(path string) string { + if strings.Contains(path, "/coverage") { + return "coverage" + } if strings.Contains(path, "/geocode/") { return "geocode" } diff --git a/services/auth_service.go b/services/auth_service.go index 4316d9d..c1da7dd 100644 --- a/services/auth_service.go +++ b/services/auth_service.go @@ -1261,6 +1261,15 @@ func (as *AuthService) HasPermission(apiKey *models.APIKey, endpoint string) boo "admin": "admin", } + // Coverage is service metadata -- which states hold data -- and returns no + // address, ZIP or boundary content. Gating it on a scope would 403 every + // key already issued, since none of them carry a "coverage" permission, + // and the endpoint exists precisely so a caller can find out what to ask + // for before asking. A valid key is still required. + if endpoint == "coverage" { + return true + } + requiredPermission, exists := permissionMap[endpoint] if !exists { return false // Unknown endpoint diff --git a/services/coverage_integration_test.go b/services/coverage_integration_test.go new file mode 100644 index 0000000..9453355 --- /dev/null +++ b/services/coverage_integration_test.go @@ -0,0 +1,112 @@ +package services + +import ( + "testing" + "time" +) + +// The endpoint exists so a caller can find out that, say, Michigan has no +// street data -- instead of querying it, getting nothing, and filing a ticket. +func TestCoverageReportsWhatIsLoaded(t *testing.T) { + db := setupCountTestDB(t) + ResetCoverageCache() + + snapshot, err := GetCoverage(db) + if err != nil { + t.Fatalf("coverage: %v", err) + } + if len(snapshot.States) == 0 { + t.Fatal("no states reported for a seeded table") + } + + var total int + for _, s := range snapshot.States { + if s.State == "" { + t.Error("a state row came back with no state code") + } + if s.Counties <= 0 { + t.Errorf("state %s reports %d counties", s.State, s.Counties) + } + total += s.Addresses + } + if total != snapshot.TotalRows { + t.Errorf("per-state addresses sum to %d, total says %d", total, snapshot.TotalRows) + } + t.Logf("coverage: %d state(s), %d addresses", len(snapshot.States), snapshot.TotalRows) +} + +// The alternative to caching is a GROUP BY over every address row on every +// request, which is the mistake the address search count query was making. +func TestCoverageIsCachedAndInvalidatedOnImport(t *testing.T) { + db := setupCountTestDB(t) + ResetCoverageCache() + + first, err := GetCoverage(db) + if err != nil { + t.Fatalf("first: %v", err) + } + + second, err := GetCoverage(db) + if err != nil { + t.Fatalf("second: %v", err) + } + if !first.GeneratedAt.Equal(second.GeneratedAt) { + t.Error("second call recomputed instead of serving the cached snapshot") + } + + // An import is the only thing that changes coverage. + time.Sleep(2 * time.Millisecond) + ResetCoverageCache() + third, err := GetCoverage(db) + if err != nil { + t.Fatalf("third: %v", err) + } + if !third.GeneratedAt.After(second.GeneratedAt) { + t.Error("snapshot was not rebuilt after the cache was invalidated") + } +} + +// The drill-down has to agree with the summary, or the two views contradict +// each other and neither can be trusted. +func TestStateCoverageAgreesWithTheSummary(t *testing.T) { + db := setupCountTestDB(t) + ResetCoverageCache() + + snapshot, err := GetCoverage(db) + if err != nil { + t.Fatalf("coverage: %v", err) + } + + for _, s := range snapshot.States { + counties, err := GetStateCoverage(db, s.State) + if err != nil { + t.Fatalf("state coverage for %s: %v", s.State, err) + } + if len(counties) != s.Counties { + t.Errorf("state %s: summary says %d counties, drill-down returns %d", + s.State, s.Counties, len(counties)) + } + + var sum int + for _, c := range counties { + sum += c.Addresses + } + if sum != s.Addresses { + t.Errorf("state %s: counties sum to %d, summary says %d", s.State, sum, s.Addresses) + } + } +} + +// An unknown state is an empty list, not an error -- that is the answer the +// caller came for. +func TestUnknownStateReturnsEmptyNotError(t *testing.T) { + db := setupCountTestDB(t) + + counties, err := GetStateCoverage(db, "MI") + if err != nil { + t.Fatalf("unknown state should not error: %v", err) + } + if len(counties) != 0 { + t.Errorf("got %d counties for an unloaded state", len(counties)) + } +} diff --git a/services/coverage_service.go b/services/coverage_service.go new file mode 100644 index 0000000..0f759d6 --- /dev/null +++ b/services/coverage_service.go @@ -0,0 +1,162 @@ +package services + +import ( + "database/sql" + "sync" + "time" + + "geocoding-api/database" +) + +// coverageTTL is how long a computed snapshot is served before it is rebuilt. +// +// Coverage only changes when a dataset is imported, which is a deliberate +// admin action measured in weeks, so a stale answer is never far wrong. The +// TTL exists because the alternative is a GROUP BY over every address row on +// every request -- the same mistake the count query in address search was +// making. +const coverageTTL = 10 * time.Minute + +// StateCoverage is what the service holds for one state. +type StateCoverage struct { + State string `json:"state"` + Counties int `json:"counties"` + Addresses int `json:"addresses"` +} + +// CountyCoverage is the per-county breakdown within a state. +type CountyCoverage struct { + County string `json:"county"` + Addresses int `json:"addresses"` +} + +// Coverage answers "what data do you actually have". +// +// Without it a caller querying a state that was never loaded gets an empty +// result set, which is indistinguishable from a bad query or a broken service. +// That is a support ticket every time. The datasets table is admin-only and +// tracks uploads rather than contents, so it cannot answer this: the original +// Ohio import did not come through the uploader and would be missing entirely. +type Coverage struct { + States []StateCoverage `json:"states"` + TotalRows int `json:"total_addresses"` + GeneratedAt time.Time `json:"generated_at"` + MaxAgeSecs int `json:"max_age_seconds"` +} + +type coverageCache struct { + mu sync.Mutex + snapshot *Coverage + builtAt time.Time +} + +var coverage = &coverageCache{} + +// GetCoverage returns the current snapshot, recomputing it when stale. +// +// The lock is held across the query on purpose. A thundering herd of requests +// arriving on a cold cache would otherwise each run the aggregate; holding it +// means the first one pays and the rest wait for that result. +func GetCoverage(db *sql.DB) (*Coverage, error) { + coverage.mu.Lock() + defer coverage.mu.Unlock() + + if coverage.snapshot != nil && time.Since(coverage.builtAt) < coverageTTL { + return coverage.snapshot, nil + } + + snapshot, err := buildCoverage(db) + if err != nil { + // Serve a stale snapshot rather than an error. Coverage is + // descriptive metadata; last week's answer is far more useful than a + // 500, and it is never badly wrong. + if coverage.snapshot != nil { + return coverage.snapshot, nil + } + return nil, err + } + + coverage.snapshot = snapshot + coverage.builtAt = time.Now() + return snapshot, nil +} + +func buildCoverage(db *sql.DB) (*Coverage, error) { + if db == nil { + db = database.DB + } + + rows, err := db.Query(` + SELECT COALESCE(NULLIF(region, ''), 'OH') AS state, + COUNT(DISTINCT county) AS counties, + COUNT(*) AS addresses + FROM ohio_addresses + GROUP BY COALESCE(NULLIF(region, ''), 'OH') + ORDER BY COUNT(*) DESC + `) + if err != nil { + return nil, err + } + defer rows.Close() + + snapshot := &Coverage{ + States: []StateCoverage{}, + GeneratedAt: time.Now(), + MaxAgeSecs: int(coverageTTL.Seconds()), + } + + for rows.Next() { + var sc StateCoverage + if err := rows.Scan(&sc.State, &sc.Counties, &sc.Addresses); err != nil { + return nil, err + } + snapshot.TotalRows += sc.Addresses + snapshot.States = append(snapshot.States, sc) + } + if err := rows.Err(); err != nil { + return nil, err + } + + return snapshot, nil +} + +// GetStateCoverage returns the per-county breakdown for one state. +// +// Not cached: it is the drill-down, asked for far less often than the summary, +// and it is already narrowed by state. +func GetStateCoverage(db *sql.DB, state string) ([]CountyCoverage, error) { + if db == nil { + db = database.DB + } + + rows, err := db.Query(` + SELECT county, COUNT(*) AS addresses + FROM ohio_addresses + WHERE COALESCE(NULLIF(region, ''), 'OH') = UPPER($1) + AND county <> '' + GROUP BY county + ORDER BY county + `, state) + if err != nil { + return nil, err + } + defer rows.Close() + + counties := []CountyCoverage{} + for rows.Next() { + var cc CountyCoverage + if err := rows.Scan(&cc.County, &cc.Addresses); err != nil { + return nil, err + } + counties = append(counties, cc) + } + return counties, rows.Err() +} + +// ResetCoverageCache drops the cached snapshot. Called after an import so the +// next request reflects newly loaded data rather than waiting out the TTL. +func ResetCoverageCache() { + coverage.mu.Lock() + defer coverage.mu.Unlock() + coverage.snapshot = nil +} diff --git a/services/dataset_service.go b/services/dataset_service.go index fc5ef87..8408fd5 100644 --- a/services/dataset_service.go +++ b/services/dataset_service.go @@ -426,6 +426,10 @@ func (s *DatasetService) ProcessGeoJSONDataset(datasetID int) error { dataset.FilePath, importer.failed) } + // An import is the only thing that changes coverage, so drop the cached + // snapshot instead of serving a stale one for the rest of the TTL. + ResetCoverageCache() + log.Printf("Successfully processed dataset %d: %d records imported, %d duplicates skipped, %d failed", datasetID, recordCount, skippedDuplicates, importer.failed) return nil