From e2a633a1f08dfec2430d032c1d10b71ddac708fe Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 15:38:32 +0100 Subject: [PATCH 01/29] =?UTF-8?q?feat(api):=20fort=20map-data=20=E2=80=94?= =?UTF-8?q?=20scan/available/by-id,=20whole-record=20results,=20dnf=20filt?= =?UTF-8?q?ers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serves fort map data from the in-memory index so map consumers stop running per-pan SQL: - Pokestop scan gains invasions via a string fetch handle on FortLookupIncident (whole IncidentData rows from incidentCache, read-through on miss), gated by a with_incidents body flag. - New GET /api/station/id/{id}; station preload under bare fort_in_memory. - New GET /api/gym/available and /api/station/available aggregates (teams/raids and battles), mirroring /api/pokestop/available. - Api results carry the WHOLE record: every persisted DB column is exposed (locked by a reflection completeness test), JSON-blob fields (guarding_pokemon_display, defenders, quest_rewards) are native JSON with documented schemas, and the generated quest columns (quest_reward_type/item_id/reward_amount/pokemon_id/pokemon_form_id + alt) are included — consumers build filter keys from them. - DNF filter additions: stationed_gmax (stations with gigantamax placed) and station_active (end_time in the future — stations are the one ephemeral fort type and expired ones otherwise dominate a match-all scan), both now-gated like the existing raid/lure/incident/contest checks. Co-Authored-By: Claude Fable 5 --- decoder/api_completeness_test.go | 82 ++++++ decoder/api_fort.go | 33 ++- decoder/api_fort_dnf_gmax_test.go | 21 ++ decoder/api_fort_dnf_station_active_test.go | 38 +++ decoder/api_gym.go | 211 +++++++++++---- decoder/api_gym_available.go | 71 ++++++ decoder/api_gym_available_test.go | 50 ++++ decoder/api_gym_test.go | 19 +- decoder/api_pokestop.go | 269 +++++++++++++------- decoder/api_pokestop_incidents_test.go | 50 ++++ decoder/api_pokestop_test.go | 31 ++- decoder/api_station.go | 54 ++-- decoder/api_station_available.go | 68 +++++ decoder/api_station_available_test.go | 39 +++ decoder/api_station_test.go | 9 +- decoder/fortRtree.go | 23 +- decoder/fort_incident_id_test.go | 34 +++ decoder/preload.go | 16 +- decoder/station_battle.go | 3 + huma_routes_test.go | 74 +++++- routes_huma.go | 96 ++++++- 21 files changed, 1095 insertions(+), 196 deletions(-) create mode 100644 decoder/api_completeness_test.go create mode 100644 decoder/api_fort_dnf_gmax_test.go create mode 100644 decoder/api_fort_dnf_station_active_test.go create mode 100644 decoder/api_gym_available.go create mode 100644 decoder/api_gym_available_test.go create mode 100644 decoder/api_pokestop_incidents_test.go create mode 100644 decoder/api_station_available.go create mode 100644 decoder/api_station_available_test.go create mode 100644 decoder/fort_incident_id_test.go diff --git a/decoder/api_completeness_test.go b/decoder/api_completeness_test.go new file mode 100644 index 00000000..3efa29b7 --- /dev/null +++ b/decoder/api_completeness_test.go @@ -0,0 +1,82 @@ +package decoder + +import ( + "reflect" + "strings" + "testing" +) + +// collectDbColumns walks a struct type — recursing into embedded anonymous +// structs (e.g. Pokestop embeds PokestopData) — and returns every persisted DB +// column: the `db:"..."` tag value, excluding db:"-" (memory-only) fields. +func collectDbColumns(t reflect.Type) []string { + var cols []string + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.Anonymous && f.Type.Kind() == reflect.Struct { + cols = append(cols, collectDbColumns(f.Type)...) + continue + } + name := strings.Split(f.Tag.Get("db"), ",")[0] + if name == "" || name == "-" { + continue + } + cols = append(cols, name) + } + return cols +} + +// collectJsonFields returns the set of top-level json field names on a struct +// (stripping ,omitempty), skipping json:"-" and untagged fields. +func collectJsonFields(t reflect.Type) map[string]bool { + m := make(map[string]bool) + for i := 0; i < t.NumField(); i++ { + name := strings.Split(t.Field(i).Tag.Get("json"), ",")[0] + if name == "" || name == "-" { + continue + } + m[name] = true + } + return m +} + +// TestApiResultsExposeEveryDbColumn locks the whole-record invariant: every +// persisted DB column of a fort record must be exposed on its API result +// struct. Adding a DB column without exposing it fails here rather than +// silently dropping data downstream — e.g. a ReactMap filter key that reads a +// column the API never sent (the quest_item_id bug that motivated this test). +// +// If a column is deliberately internal, add it to that case's `allow` set WITH +// a comment explaining why — do not weaken the assertion. +func TestApiResultsExposeEveryDbColumn(t *testing.T) { + cases := []struct { + name string + dbType reflect.Type + apiType reflect.Type + allow map[string]bool // db columns intentionally not exposed (with reason) + }{ + {"pokestop", reflect.TypeOf(Pokestop{}), reflect.TypeOf(ApiPokestopResult{}), nil}, + {"station", reflect.TypeOf(Station{}), reflect.TypeOf(ApiStationResult{}), nil}, + {"gym", reflect.TypeOf(Gym{}), reflect.TypeOf(ApiGymResult{}), nil}, + {"incident", reflect.TypeOf(Incident{}), reflect.TypeOf(ApiPokestopIncident{}), nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cols := collectDbColumns(c.dbType) + if len(cols) == 0 { + t.Fatalf("no db columns found for %s — reflection is broken, the test would pass vacuously", c.dbType.Name()) + } + exposed := collectJsonFields(c.apiType) + for _, col := range cols { + if c.allow[col] { + continue + } + if !exposed[col] { + t.Errorf("DB column %q of %s is not exposed as a json field on %s.\n"+ + "Add `json:%q` to %s, or if it is intentionally internal add it to this case's allow set with a reason.", + col, c.dbType.Name(), c.apiType.Name(), col, c.apiType.Name()) + } + } + }) + } +} diff --git a/decoder/api_fort.go b/decoder/api_fort.go index 8413f4ac..2b8c496b 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -12,10 +12,11 @@ import ( ) type ApiFortScan struct { - Min ApiLatLon `json:"min" doc:"SW (minimum lat/lon) corner of the bounding box."` - Max ApiLatLon `json:"max" doc:"NE (maximum lat/lon) corner of the bounding box."` - Limit int `json:"limit" required:"false" doc:"Max results to return; 0 uses the server default."` - DnfFilters []ApiFortDnfFilter `json:"filters" required:"false" doc:"OR'd filter clauses; a fort matches if it satisfies any one clause. List conditions apply only when present: omit or send null for no constraint — an explicitly empty list matches nothing."` + Min ApiLatLon `json:"min" doc:"SW (minimum lat/lon) corner of the bounding box."` + Max ApiLatLon `json:"max" doc:"NE (maximum lat/lon) corner of the bounding box."` + Limit int `json:"limit" required:"false" doc:"Max results to return; 0 uses the server default."` + DnfFilters []ApiFortDnfFilter `json:"filters" required:"false" doc:"OR'd filter clauses; a fort matches if it satisfies any one clause. List conditions apply only when present: omit or send null for no constraint — an explicitly empty list matches nothing."` + WithIncidents bool `json:"with_incidents" required:"false" doc:"Pokestop only: when true, each pokestop result includes its active incidents (invasions). Ignored for gym/station."` } type ApiFortDnfFilter struct { @@ -49,6 +50,8 @@ type ApiFortDnfFilter struct { // Station BattleLevel []int8 `json:"battle_level" required:"false" doc:"Station only: allowed active max battle levels; omitted or null means no battle level constraint. Only matches stations with an active battle."` BattlePokemon []ApiDnfId `json:"battle_pokemon" required:"false" doc:"Station only: allowed active max battle pokemon/form pairs; omitted or null means no battle pokemon constraint. Only matches stations with an active battle."` + StationedGmax *bool `json:"stationed_gmax" required:"false" doc:"Station only: when true, only match stations with at least one stationed Gigantamax pokemon; null means no constraint."` + StationActive *bool `json:"station_active" required:"false" doc:"Station only: when true, only match stations whose end_time is in the future (still present); when false, only expired stations. Stations are the one ephemeral fort type — expired ones accumulate in the index. Null means no constraint."` } type ApiDnfId struct { @@ -213,6 +216,12 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn } } case STATION: + if filter.StationActive != nil && *filter.StationActive != (fortLookup.StationEndTimestamp > now) { + return false + } + if filter.StationedGmax != nil && *filter.StationedGmax && fortLookup.TotalStationedGmax <= 0 { + return false + } if filter.BattleLevel != nil || filter.BattlePokemon != nil { if len(fortLookup.StationBattles) == 0 { if fortLookup.BattleEndTimestamp <= now { @@ -345,11 +354,19 @@ func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails returnKeys, examined, skipped, total := internalGetForts(POKESTOP, retrieveParameters) results := make([]*ApiPokestopResult, 0, len(returnKeys)) start := time.Now() + now := time.Now().Unix() for _, key := range returnKeys { pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanpokemon") if err == nil && pokestop != nil { pokestopCopy := buildPokestopResult(pokestop) + if unlock != nil { + unlock() // release pokestop lock BEFORE locking incidents (lock-order) + unlock = nil + } + if retrieveParameters.WithIncidents { + pokestopCopy.Invasions = CollectPokestopIncidents(context.Background(), dbDetails, key, now) + } results = append(results, &pokestopCopy) } if unlock != nil { @@ -394,6 +411,7 @@ func StationScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) func FortCombinedScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiFortCombinedScanResult { gymKeys, pokestopKeys, stationKeys, examined, skipped, total := internalGetFortsCombined(retrieveParameters) start := time.Now() + now := time.Now().Unix() gyms := make([]*ApiGymResult, 0, len(gymKeys)) for _, key := range gymKeys { @@ -412,6 +430,13 @@ func FortCombinedScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDet pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanpokemonPokemon") if err == nil && pokestop != nil { pokestopCopy := buildPokestopResult(pokestop) + if unlock != nil { + unlock() // release pokestop lock BEFORE locking incidents (lock-order) + unlock = nil + } + if retrieveParameters.WithIncidents { + pokestopCopy.Invasions = CollectPokestopIncidents(context.Background(), dbDetails, key, now) + } pokestops = append(pokestops, &pokestopCopy) } if unlock != nil { diff --git a/decoder/api_fort_dnf_gmax_test.go b/decoder/api_fort_dnf_gmax_test.go new file mode 100644 index 00000000..13b34fd2 --- /dev/null +++ b/decoder/api_fort_dnf_gmax_test.go @@ -0,0 +1,21 @@ +package decoder + +import "testing" + +func TestIsFortDnfMatch_StationedGmax(t *testing.T) { + gmax := true + withGmax := &FortLookup{FortType: STATION, TotalStationedGmax: 3} + noGmax := &FortLookup{FortType: STATION, TotalStationedGmax: 0} + now := int64(1000) + + if !isFortDnfMatch(STATION, withGmax, &ApiFortDnfFilter{StationedGmax: &gmax}, now) { + t.Error("station with stationed gmax should match stationed_gmax:true") + } + if isFortDnfMatch(STATION, noGmax, &ApiFortDnfFilter{StationedGmax: &gmax}, now) { + t.Error("station without stationed gmax must not match stationed_gmax:true") + } + // null gmax filter is a wildcard — matches either + if !isFortDnfMatch(STATION, noGmax, &ApiFortDnfFilter{}, now) { + t.Error("no stationed_gmax constraint should match any station") + } +} diff --git a/decoder/api_fort_dnf_station_active_test.go b/decoder/api_fort_dnf_station_active_test.go new file mode 100644 index 00000000..269f9229 --- /dev/null +++ b/decoder/api_fort_dnf_station_active_test.go @@ -0,0 +1,38 @@ +package decoder + +import "testing" + +// TestIsFortDnfMatch_StationActive locks the station liveness gate: stations +// are the one ephemeral fort type, and expired ones accumulate in the index — +// station_active:true matches only stations whose end_time is in the future. +func TestIsFortDnfMatch_StationActive(t *testing.T) { + active := true + inactive := false + now := int64(1000) + live := FortLookup{FortType: STATION, StationEndTimestamp: 2000} + dead := FortLookup{FortType: STATION, StationEndTimestamp: 500} + + if !isFortDnfMatch(STATION, &live, &ApiFortDnfFilter{StationActive: &active}, now) { + t.Error("live station should match station_active:true") + } + if isFortDnfMatch(STATION, &dead, &ApiFortDnfFilter{StationActive: &active}, now) { + t.Error("expired station must not match station_active:true") + } + if !isFortDnfMatch(STATION, &dead, &ApiFortDnfFilter{StationActive: &inactive}, now) { + t.Error("expired station should match station_active:false") + } + if !isFortDnfMatch(STATION, &dead, &ApiFortDnfFilter{}, now) { + t.Error("no station_active constraint should match any station") + } + // composes with gmax within a clause (AND) + gmax := true + liveGmax := FortLookup{FortType: STATION, StationEndTimestamp: 2000, TotalStationedGmax: 2} + deadGmax := FortLookup{FortType: STATION, StationEndTimestamp: 500, TotalStationedGmax: 2} + f := ApiFortDnfFilter{StationActive: &active, StationedGmax: &gmax} + if !isFortDnfMatch(STATION, &liveGmax, &f, now) { + t.Error("live gmax station should match combined clause") + } + if isFortDnfMatch(STATION, &deadGmax, &f, now) { + t.Error("expired gmax station must not match combined clause") + } +} diff --git a/decoder/api_gym.go b/decoder/api_gym.go index f704b580..29770825 100644 --- a/decoder/api_gym.go +++ b/decoder/api_gym.go @@ -2,59 +2,182 @@ package decoder import ( "context" + "encoding/json" "fmt" "math" + "reflect" "strings" + "github.com/danielgtaylor/huma/v2" + "github.com/guregu/null/v6" + "golbat/db" "golbat/geo" ) +// ApiGymGuardingPokemon is the display detail of the pokemon guarding a gym. +// It is doc-only: used purely for OpenAPI schema reflection (see +// ApiGymGuardingPokemonRaw.Schema) and never unmarshaled into. The wire value +// is passed through verbatim from the stored JSON blob. +type ApiGymGuardingPokemon struct { + Form int `json:"form,omitempty" doc:"Form id"` + Costume int `json:"costume,omitempty" doc:"Costume id"` + Gender int `json:"gender" doc:"Gender"` + Shiny bool `json:"shiny,omitempty" doc:"Shiny"` + TempEvolution int `json:"temp_evolution,omitempty" doc:"Temp (mega) evolution id"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty" doc:"Temp evolution finish (ms)"` + Alignment int `json:"alignment,omitempty" doc:"Alignment (shadow/purified)"` + Badge int `json:"badge,omitempty" doc:"Pokemon badge"` + Background *int64 `json:"background,omitempty" doc:"Background id"` +} + +// ApiGymDefender is one pokemon defending a gym. It is doc-only: used purely +// for OpenAPI schema reflection (see ApiGymDefendersRaw.Schema) and never +// unmarshaled into. The wire value is passed through verbatim from the stored +// JSON blob. MotivationNow is a plain float64 here (rather than +// util.RoundedFloat4) since this struct is never marshaled for the wire — the +// two types produce an identical `number` schema. +type ApiGymDefender struct { + PokemonId int `json:"pokemon_id,omitempty" doc:"Defender pokedex id"` + Form int `json:"form,omitempty" doc:"Form id"` + Costume int `json:"costume,omitempty" doc:"Costume id"` + Gender int `json:"gender" doc:"Gender"` + Shiny bool `json:"shiny,omitempty" doc:"Shiny"` + TempEvolution int `json:"temp_evolution,omitempty" doc:"Temp evolution id"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty" doc:"Temp evolution finish (ms)"` + Alignment int `json:"alignment,omitempty" doc:"Alignment"` + Badge int `json:"badge,omitempty" doc:"Badge"` + Background *int64 `json:"background,omitempty" doc:"Background id"` + DeployedMs int64 `json:"deployed_ms,omitempty" doc:"Deployment duration (ms)"` + DeployedTime int64 `json:"deployed_time,omitempty" doc:"Approx unix deploy time"` + BattlesWon int32 `json:"battles_won" doc:"Battles won"` + BattlesLost int32 `json:"battles_lost" doc:"Battles lost"` + TimesFed int32 `json:"times_fed" doc:"Times fed"` + MotivationNow float64 `json:"motivation_now" doc:"Current motivation"` + CpNow int32 `json:"cp_now" doc:"Current CP"` + CpWhenDeployed int32 `json:"cp_when_deployed" doc:"CP when deployed"` +} + +// jsonRaw wraps a pre-serialized JSON blob stored as a null.String column so it +// is emitted as native JSON instead of an escaped string. Returns nil (JSON +// null) when the column is unset, empty, or not valid JSON. +func jsonRaw(s null.String) *json.RawMessage { + if !s.Valid || s.String == "" || !json.Valid([]byte(s.String)) { + return nil + } + r := json.RawMessage(s.String) + return &r +} + +// ApiGymGuardingPokemonRaw carries the stored guarding_pokemon_display JSON +// verbatim (no decode/re-encode). Schema() advertises the ApiGymGuardingPokemon +// shape to OpenAPI; the wire bytes are passed through unchanged. +type ApiGymGuardingPokemonRaw json.RawMessage + +// MarshalJSON returns the wrapped bytes verbatim. This must be defined +// explicitly: a named json.RawMessage type does not inherit RawMessage's +// MarshalJSON, so without this override encoding/json would base64-encode it +// as a byte slice instead of passing it through as JSON. +func (m ApiGymGuardingPokemonRaw) MarshalJSON() ([]byte, error) { + if len(m) == 0 { + return []byte("null"), nil + } + return m, nil +} + +// Schema implements huma.SchemaProvider, documenting the real +// ApiGymGuardingPokemon shape even though the wire value is passed through raw. +func (ApiGymGuardingPokemonRaw) Schema(r huma.Registry) *huma.Schema { + return r.Schema(reflect.TypeOf(ApiGymGuardingPokemon{}), true, "ApiGymGuardingPokemon") +} + +// ApiGymDefendersRaw carries the stored defenders JSON array verbatim (no +// decode/re-encode). Schema() advertises the []ApiGymDefender shape to +// OpenAPI; the wire bytes are passed through unchanged. +type ApiGymDefendersRaw json.RawMessage + +// MarshalJSON returns the wrapped bytes verbatim (see +// ApiGymGuardingPokemonRaw.MarshalJSON for why this override is required). +func (m ApiGymDefendersRaw) MarshalJSON() ([]byte, error) { + if len(m) == 0 { + return []byte("null"), nil + } + return m, nil +} + +// Schema implements huma.SchemaProvider, documenting the real +// []ApiGymDefender shape even though the wire value is passed through raw. +func (ApiGymDefendersRaw) Schema(r huma.Registry) *huma.Schema { + return r.Schema(reflect.TypeOf([]ApiGymDefender{}), true, "ApiGymDefender") +} + +// gymGuardingRaw returns the pre-serialized guarding_pokemon_display blob +// stored on the Gym record as a verbatim raw-JSON passthrough value. Returns +// nil (JSON null) when the column is unset, empty, or not valid JSON, so we +// never emit invalid JSON verbatim. +func gymGuardingRaw(s null.String) ApiGymGuardingPokemonRaw { + if !s.Valid || s.String == "" || !json.Valid([]byte(s.String)) { + return nil + } + return ApiGymGuardingPokemonRaw(s.String) +} + +// gymDefendersRaw returns the pre-serialized defenders blob stored on the Gym +// record as a verbatim raw-JSON passthrough value. Returns nil (JSON null) +// when the column is unset, empty, or not valid JSON, so we never emit +// invalid JSON verbatim. +func gymDefendersRaw(s null.String) ApiGymDefendersRaw { + if !s.Valid || s.String == "" || !json.Valid([]byte(s.String)) { + return nil + } + return ApiGymDefendersRaw(s.String) +} + // ApiGymResult is the API representation of a gym. Nullable database columns are // represented as pointers (nil => JSON null) without omitempty so every key is // always present. type ApiGymResult struct { - Id string `json:"id" doc:"Fort ID of the gym"` - Lat float64 `json:"lat" doc:"Latitude of the gym"` - Lon float64 `json:"lon" doc:"Longitude of the gym"` - Name *string `json:"name" doc:"Name of the gym"` - Url *string `json:"url" doc:"Image URL of the gym"` - LastModifiedTimestamp *int64 `json:"last_modified_timestamp" doc:"Unix timestamp when the gym was last modified in-game"` - RaidEndTimestamp *int64 `json:"raid_end_timestamp" doc:"Unix timestamp when the current raid ends"` - RaidSpawnTimestamp *int64 `json:"raid_spawn_timestamp" doc:"Unix timestamp when the current raid egg spawned"` - RaidBattleTimestamp *int64 `json:"raid_battle_timestamp" doc:"Unix timestamp when the current raid battle begins"` - Updated int64 `json:"updated" doc:"Unix timestamp when the record was last updated"` - RaidPokemonId *int64 `json:"raid_pokemon_id" doc:"Pokedex ID of the raid boss"` - GuardingPokemonId *int64 `json:"guarding_pokemon_id" doc:"Pokedex ID of the pokemon guarding the gym"` - GuardingPokemonDisplay *string `json:"guarding_pokemon_display" doc:"Display details of the guarding pokemon"` - AvailableSlots *int64 `json:"available_slots" doc:"Number of open defender slots"` - TeamId *int64 `json:"team_id" doc:"ID of the team controlling the gym"` - RaidLevel *int64 `json:"raid_level" doc:"Level/tier of the current raid"` - Enabled *int64 `json:"enabled" doc:"Whether the gym is enabled"` - ExRaidEligible *int64 `json:"ex_raid_eligible" doc:"Whether the gym is eligible for EX raids"` - InBattle *int64 `json:"in_battle" doc:"Whether the gym is currently in battle"` - RaidPokemonMove1 *int64 `json:"raid_pokemon_move_1" doc:"Fast move ID of the raid boss"` - RaidPokemonMove2 *int64 `json:"raid_pokemon_move_2" doc:"Charge move ID of the raid boss"` - RaidPokemonForm *int64 `json:"raid_pokemon_form" doc:"Form ID of the raid boss"` - RaidPokemonAlignment *int64 `json:"raid_pokemon_alignment" doc:"Alignment of the raid boss"` - RaidPokemonCp *int64 `json:"raid_pokemon_cp" doc:"Combat power of the raid boss"` - RaidIsExclusive *int64 `json:"raid_is_exclusive" doc:"Whether the current raid is exclusive (EX)"` - CellId *int64 `json:"cell_id" doc:"S2 cell ID the gym belongs to"` - Deleted bool `json:"deleted" doc:"Whether the gym has been deleted"` - TotalCp *int64 `json:"total_cp" doc:"Total combat power of the gym defenders"` - FirstSeenTimestamp int64 `json:"first_seen_timestamp" doc:"Unix timestamp when the gym was first seen"` - RaidPokemonGender *int64 `json:"raid_pokemon_gender" doc:"Gender of the raid boss"` - SponsorId *int64 `json:"sponsor_id" doc:"Sponsor ID of the gym, if sponsored"` - PartnerId *string `json:"partner_id" doc:"Partner ID of the gym, if partnered"` - RaidPokemonCostume *int64 `json:"raid_pokemon_costume" doc:"Costume ID of the raid boss"` - RaidPokemonEvolution *int64 `json:"raid_pokemon_evolution" doc:"Evolution ID of the raid boss (e.g. mega)"` - ArScanEligible *int64 `json:"ar_scan_eligible" doc:"Whether the gym is eligible for AR scanning"` - PowerUpLevel *int64 `json:"power_up_level" doc:"Power-up level of the gym"` - PowerUpPoints *int64 `json:"power_up_points" doc:"Power-up points accumulated for the gym"` - PowerUpEndTimestamp *int64 `json:"power_up_end_timestamp" doc:"Unix timestamp when the power-up ends"` - Description *string `json:"description" doc:"Description of the gym"` - Defenders *string `json:"defenders" doc:"Serialized defender pokemon data"` - Rsvps *string `json:"rsvps" doc:"Serialized raid RSVP data"` + Id string `json:"id" doc:"Fort ID of the gym"` + Lat float64 `json:"lat" doc:"Latitude of the gym"` + Lon float64 `json:"lon" doc:"Longitude of the gym"` + Name *string `json:"name" doc:"Name of the gym"` + Url *string `json:"url" doc:"Image URL of the gym"` + LastModifiedTimestamp *int64 `json:"last_modified_timestamp" doc:"Unix timestamp when the gym was last modified in-game"` + RaidEndTimestamp *int64 `json:"raid_end_timestamp" doc:"Unix timestamp when the current raid ends"` + RaidSpawnTimestamp *int64 `json:"raid_spawn_timestamp" doc:"Unix timestamp when the current raid egg spawned"` + RaidBattleTimestamp *int64 `json:"raid_battle_timestamp" doc:"Unix timestamp when the current raid battle begins"` + Updated int64 `json:"updated" doc:"Unix timestamp when the record was last updated"` + RaidPokemonId *int64 `json:"raid_pokemon_id" doc:"Pokedex ID of the raid boss"` + GuardingPokemonId *int64 `json:"guarding_pokemon_id" doc:"Pokedex ID of the pokemon guarding the gym"` + GuardingPokemonDisplay ApiGymGuardingPokemonRaw `json:"guarding_pokemon_display" doc:"Display details of the guarding pokemon"` + AvailableSlots *int64 `json:"available_slots" doc:"Number of open defender slots"` + TeamId *int64 `json:"team_id" doc:"ID of the team controlling the gym"` + RaidLevel *int64 `json:"raid_level" doc:"Level/tier of the current raid"` + Enabled *int64 `json:"enabled" doc:"Whether the gym is enabled"` + ExRaidEligible *int64 `json:"ex_raid_eligible" doc:"Whether the gym is eligible for EX raids"` + InBattle *int64 `json:"in_battle" doc:"Whether the gym is currently in battle"` + RaidPokemonMove1 *int64 `json:"raid_pokemon_move_1" doc:"Fast move ID of the raid boss"` + RaidPokemonMove2 *int64 `json:"raid_pokemon_move_2" doc:"Charge move ID of the raid boss"` + RaidPokemonForm *int64 `json:"raid_pokemon_form" doc:"Form ID of the raid boss"` + RaidPokemonAlignment *int64 `json:"raid_pokemon_alignment" doc:"Alignment of the raid boss"` + RaidPokemonCp *int64 `json:"raid_pokemon_cp" doc:"Combat power of the raid boss"` + RaidIsExclusive *int64 `json:"raid_is_exclusive" doc:"Whether the current raid is exclusive (EX)"` + CellId *int64 `json:"cell_id" doc:"S2 cell ID the gym belongs to"` + Deleted bool `json:"deleted" doc:"Whether the gym has been deleted"` + TotalCp *int64 `json:"total_cp" doc:"Total combat power of the gym defenders"` + FirstSeenTimestamp int64 `json:"first_seen_timestamp" doc:"Unix timestamp when the gym was first seen"` + RaidPokemonGender *int64 `json:"raid_pokemon_gender" doc:"Gender of the raid boss"` + SponsorId *int64 `json:"sponsor_id" doc:"Sponsor ID of the gym, if sponsored"` + PartnerId *string `json:"partner_id" doc:"Partner ID of the gym, if partnered"` + RaidPokemonCostume *int64 `json:"raid_pokemon_costume" doc:"Costume ID of the raid boss"` + RaidPokemonEvolution *int64 `json:"raid_pokemon_evolution" doc:"Evolution ID of the raid boss (e.g. mega)"` + ArScanEligible *int64 `json:"ar_scan_eligible" doc:"Whether the gym is eligible for AR scanning"` + PowerUpLevel *int64 `json:"power_up_level" doc:"Power-up level of the gym"` + PowerUpPoints *int64 `json:"power_up_points" doc:"Power-up points accumulated for the gym"` + PowerUpEndTimestamp *int64 `json:"power_up_end_timestamp" doc:"Unix timestamp when the power-up ends"` + Description *string `json:"description" doc:"Description of the gym"` + Defenders ApiGymDefendersRaw `json:"defenders" doc:"Defender pokemon"` + Rsvps *json.RawMessage `json:"rsvps" doc:"Raid RSVP data"` } func buildGymResult(gym *Gym) ApiGymResult { @@ -71,7 +194,7 @@ func buildGymResult(gym *Gym) ApiGymResult { Updated: gym.Updated, RaidPokemonId: gym.RaidPokemonId.Ptr(), GuardingPokemonId: gym.GuardingPokemonId.Ptr(), - GuardingPokemonDisplay: gym.GuardingPokemonDisplay.Ptr(), + GuardingPokemonDisplay: gymGuardingRaw(gym.GuardingPokemonDisplay), AvailableSlots: gym.AvailableSlots.Ptr(), TeamId: gym.TeamId.Ptr(), RaidLevel: gym.RaidLevel.Ptr(), @@ -98,8 +221,8 @@ func buildGymResult(gym *Gym) ApiGymResult { PowerUpPoints: gym.PowerUpPoints.Ptr(), PowerUpEndTimestamp: gym.PowerUpEndTimestamp.Ptr(), Description: gym.Description.Ptr(), - Defenders: gym.Defenders.Ptr(), - Rsvps: gym.Rsvps.Ptr(), + Defenders: gymDefendersRaw(gym.Defenders), + Rsvps: jsonRaw(gym.Rsvps), } } diff --git a/decoder/api_gym_available.go b/decoder/api_gym_available.go new file mode 100644 index 00000000..21a7dc21 --- /dev/null +++ b/decoder/api_gym_available.go @@ -0,0 +1,71 @@ +package decoder + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +// ApiGymTeamAvailable is one distinct (team, available-slots) pair present on +// resident gyms, with how many gyms carry it. ReactMap derives its t/g keys. +type ApiGymTeamAvailable struct { + TeamId int8 `json:"team_id" doc:"Controlling team id (0 = uncontested)"` + AvailableSlots int8 `json:"available_slots" doc:"Open defender slots"` + Count int `json:"count" doc:"Number of resident gyms with this team/slots"` +} + +// ApiGymRaidAvailable is one distinct active raid option on resident gyms. +// PokemonId 0 means an egg (no boss yet). ReactMap derives its e/r/boss keys. +type ApiGymRaidAvailable struct { + RaidLevel int8 `json:"raid_level" doc:"Raid level/tier"` + PokemonId int16 `json:"pokemon_id" doc:"Raid boss pokemon id; 0 = egg (unhatched)"` + Form int16 `json:"form" doc:"Raid boss form id, else 0"` + Count int `json:"count" doc:"Number of resident gyms with this raid option"` +} + +// ApiAvailableGyms is the whole-instance gym filter snapshot served by +// GET /api/gym/available. +type ApiAvailableGyms struct { + Teams []ApiGymTeamAvailable `json:"teams" doc:"Distinct team + available-slot pairs on resident gyms"` + Raids []ApiGymRaidAvailable `json:"raids" doc:"Distinct active raid levels/bosses/eggs on resident gyms"` +} + +// GetAvailableGyms builds the gym filter snapshot from a single fortLookupCache +// range over resident gyms — no maintained map (FortLookup carries every gym +// filter field). Teams are all-resident (no time filter); raids require an +// unexpired raid with level > 0. +func GetAvailableGyms(now int64) *ApiAvailableGyms { + start := time.Now() + res := &ApiAvailableGyms{Teams: []ApiGymTeamAvailable{}, Raids: []ApiGymRaidAvailable{}} + teams := map[ApiGymTeamAvailable]int{} + raids := map[ApiGymRaidAvailable]int{} + forts := 0 + + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + if fl.FortType != GYM { + return true + } + forts++ + teams[ApiGymTeamAvailable{TeamId: fl.TeamId, AvailableSlots: fl.AvailableSlots}]++ + if fl.RaidLevel > 0 && fl.RaidEndTimestamp > now { + raids[ApiGymRaidAvailable{RaidLevel: fl.RaidLevel, PokemonId: fl.RaidPokemonId, Form: fl.RaidPokemonForm}]++ + } + return true + }) + + for k, n := range teams { + k.Count = n + res.Teams = append(res.Teams, k) + } + for k, n := range raids { + k.Count = n + res.Raids = append(res.Raids, k) + } + + if statsCollector != nil { + statsCollector.ObserveApiScan("available-gyms", time.Since(start).Seconds()) + } + log.Infof("available-gyms built in %s: scanned %d gyms -> %d team/slot, %d raid options", + time.Since(start), forts, len(res.Teams), len(res.Raids)) + return res +} diff --git a/decoder/api_gym_available_test.go b/decoder/api_gym_available_test.go new file mode 100644 index 00000000..f3118011 --- /dev/null +++ b/decoder/api_gym_available_test.go @@ -0,0 +1,50 @@ +package decoder + +import ( + "testing" + + "github.com/puzpuzpuz/xsync/v4" +) + +func TestGetAvailableGyms(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + now := int64(1_000_000) + + // gym with team + active raid boss + fortLookupCache.Store("g1", FortLookup{ + FortType: GYM, TeamId: 1, AvailableSlots: 2, + RaidLevel: 5, RaidPokemonId: 150, RaidPokemonForm: 0, RaidEndTimestamp: now + 100, + }) + // gym with an active egg (no boss) and an EXPIRED raid on another + fortLookupCache.Store("g2", FortLookup{ + FortType: GYM, TeamId: 2, AvailableSlots: 6, + RaidLevel: 3, RaidPokemonId: 0, RaidEndTimestamp: now + 100, + }) + fortLookupCache.Store("g3", FortLookup{ + FortType: GYM, TeamId: 1, AvailableSlots: 0, + RaidLevel: 5, RaidPokemonId: 999, RaidEndTimestamp: now - 1, // expired -> excluded + }) + // a pokestop must be ignored + fortLookupCache.Store("s1", FortLookup{FortType: POKESTOP, LureId: 501}) + + res := GetAvailableGyms(now) + + if len(res.Teams) != 3 { // (1,2),(2,6),(1,0) + t.Fatalf("teams: %+v", res.Teams) + } + // raids: boss 150 lvl5, egg lvl3; expired 999 excluded + var bosses, eggs int + for _, r := range res.Raids { + if r.PokemonId == 999 { + t.Fatalf("expired raid leaked: %+v", r) + } + if r.PokemonId == 0 { + eggs++ + } else { + bosses++ + } + } + if bosses != 1 || eggs != 1 { + t.Fatalf("raids: %+v", res.Raids) + } +} diff --git a/decoder/api_gym_test.go b/decoder/api_gym_test.go index 9768c1c4..c1495cc6 100644 --- a/decoder/api_gym_test.go +++ b/decoder/api_gym_test.go @@ -9,6 +9,10 @@ import ( // goldenSnapshotGym is a representative gym with a mix of set and unset (null) // fields across every nullable column, used to pin the exact wire format. +// GuardingPokemonDisplay, Defenders, and Rsvps are populated with pre-serialized +// JSON blobs (as they are stored on the DB record) so the test exercises the +// raw-JSON passthrough (the stored bytes are emitted verbatim, not decoded and +// re-encoded), not just the null case. func goldenSnapshotGym() *Gym { return &Gym{ GymData: GymData{ @@ -24,7 +28,9 @@ func goldenSnapshotGym() *Gym { Updated: 1699999999, RaidPokemonId: null.IntFrom(150), GuardingPokemonId: null.IntFrom(143), - // GuardingPokemonDisplay intentionally left null + GuardingPokemonDisplay: null.StringFrom( + `{"form":91,"costume":0,"gender":1,"shiny":true,"temp_evolution":0,"alignment":0,"badge":0,"background":7}`, + ), AvailableSlots: null.IntFrom(3), TeamId: null.IntFrom(2), RaidLevel: null.IntFrom(5), @@ -51,7 +57,9 @@ func goldenSnapshotGym() *Gym { PowerUpPoints: null.IntFrom(50), // PowerUpEndTimestamp intentionally left null Description: null.StringFrom("A test gym"), - // Defenders intentionally left null + Defenders: null.StringFrom( + `[{"pokemon_id":143,"form":0,"costume":0,"gender":1,"shiny":false,"temp_evolution":0,"alignment":1,"badge":0,"background":null,"deployed_ms":3600000,"deployed_time":1699996400,"battles_won":4,"battles_lost":1,"times_fed":2,"motivation_now":0.6667,"cp_now":2500,"cp_when_deployed":2600}]`, + ), Rsvps: null.StringFrom("[]"), }, } @@ -60,14 +68,17 @@ func goldenSnapshotGym() *Gym { // TestBuildGymResult_GoldenSnapshot pins the exact JSON wire format of an // ApiGymResult. Any accidental change to a json tag, field type, pointer/null // handling, or field order will fail this test. Unset nullable fields serialize -// as null (pointers are nil, no omitempty). +// as null (pointers are nil, no omitempty). guarding_pokemon_display and +// defenders are raw-JSON passthrough, so they appear on the wire exactly as +// stored (including zero-valued fields that a re-marshal through +// ApiGymGuardingPokemon/ApiGymDefender would have omitted via omitempty). func TestBuildGymResult_GoldenSnapshot(t *testing.T) { got, err := json.Marshal(buildGymResult(goldenSnapshotGym())) if err != nil { t.Fatalf("marshal: %v", err) } - const want = `{"id":"gym-abc","lat":12.3456,"lon":-65.4321,"name":"Test Gym","url":"https://example.com/gym.png","last_modified_timestamp":1699990000,"raid_end_timestamp":1700003600,"raid_spawn_timestamp":null,"raid_battle_timestamp":1700000000,"updated":1699999999,"raid_pokemon_id":150,"guarding_pokemon_id":143,"guarding_pokemon_display":null,"available_slots":3,"team_id":2,"raid_level":5,"enabled":1,"ex_raid_eligible":0,"in_battle":0,"raid_pokemon_move_1":216,"raid_pokemon_move_2":94,"raid_pokemon_form":0,"raid_pokemon_alignment":0,"raid_pokemon_cp":3500,"raid_is_exclusive":0,"cell_id":1234567890123,"deleted":false,"total_cp":12000,"first_seen_timestamp":1699990000,"raid_pokemon_gender":1,"sponsor_id":null,"partner_id":"partner-1","raid_pokemon_costume":0,"raid_pokemon_evolution":0,"ar_scan_eligible":1,"power_up_level":2,"power_up_points":50,"power_up_end_timestamp":null,"description":"A test gym","defenders":null,"rsvps":"[]"}` + const want = `{"id":"gym-abc","lat":12.3456,"lon":-65.4321,"name":"Test Gym","url":"https://example.com/gym.png","last_modified_timestamp":1699990000,"raid_end_timestamp":1700003600,"raid_spawn_timestamp":null,"raid_battle_timestamp":1700000000,"updated":1699999999,"raid_pokemon_id":150,"guarding_pokemon_id":143,"guarding_pokemon_display":{"form":91,"costume":0,"gender":1,"shiny":true,"temp_evolution":0,"alignment":0,"badge":0,"background":7},"available_slots":3,"team_id":2,"raid_level":5,"enabled":1,"ex_raid_eligible":0,"in_battle":0,"raid_pokemon_move_1":216,"raid_pokemon_move_2":94,"raid_pokemon_form":0,"raid_pokemon_alignment":0,"raid_pokemon_cp":3500,"raid_is_exclusive":0,"cell_id":1234567890123,"deleted":false,"total_cp":12000,"first_seen_timestamp":1699990000,"raid_pokemon_gender":1,"sponsor_id":null,"partner_id":"partner-1","raid_pokemon_costume":0,"raid_pokemon_evolution":0,"ar_scan_eligible":1,"power_up_level":2,"power_up_points":50,"power_up_end_timestamp":null,"description":"A test gym","defenders":[{"pokemon_id":143,"form":0,"costume":0,"gender":1,"shiny":false,"temp_evolution":0,"alignment":1,"badge":0,"background":null,"deployed_ms":3600000,"deployed_time":1699996400,"battles_won":4,"battles_lost":1,"times_fed":2,"motivation_now":0.6667,"cp_now":2500,"cp_when_deployed":2600}],"rsvps":[]}` if string(got) != want { t.Errorf("wire format changed.\n got: %s\nwant: %s", got, want) diff --git a/decoder/api_pokestop.go b/decoder/api_pokestop.go index 636c4164..14167d05 100644 --- a/decoder/api_pokestop.go +++ b/decoder/api_pokestop.go @@ -1,102 +1,199 @@ package decoder +import ( + "context" + "encoding/json" + + db "golbat/db" +) + // ApiPokestopResult is the API representation of a pokestop. Nullable database // columns are represented as pointers (nil => JSON null) without omitempty so // every key is always present. type ApiPokestopResult struct { - Id string `json:"id" doc:"Fort ID of the pokestop"` - Lat float64 `json:"lat" doc:"Latitude of the pokestop"` - Lon float64 `json:"lon" doc:"Longitude of the pokestop"` - Name *string `json:"name" doc:"Name of the pokestop"` - Url *string `json:"url" doc:"Image URL of the pokestop"` - LureExpireTimestamp *int64 `json:"lure_expire_timestamp" doc:"Unix timestamp when the current lure expires"` - LastModifiedTimestamp *int64 `json:"last_modified_timestamp" doc:"Unix timestamp when the pokestop was last modified in-game"` - Updated int64 `json:"updated" doc:"Unix timestamp when the record was last updated"` - Enabled *bool `json:"enabled" doc:"Whether the pokestop is enabled"` - QuestType *int64 `json:"quest_type" doc:"Type of the AR quest"` - QuestTimestamp *int64 `json:"quest_timestamp" doc:"Unix timestamp when the AR quest was set"` - QuestTarget *int64 `json:"quest_target" doc:"Target count for the AR quest"` - QuestConditions *string `json:"quest_conditions" doc:"Serialized conditions of the AR quest"` - QuestRewards *string `json:"quest_rewards" doc:"Serialized rewards of the AR quest"` - QuestTemplate *string `json:"quest_template" doc:"Template ID of the AR quest"` - QuestTitle *string `json:"quest_title" doc:"Title of the AR quest"` - QuestExpiry *int64 `json:"quest_expiry" doc:"Unix timestamp when the AR quest expires"` - CellId *int64 `json:"cell_id" doc:"S2 cell ID the pokestop belongs to"` - Deleted bool `json:"deleted" doc:"Whether the pokestop has been deleted"` - LureId int16 `json:"lure_id" doc:"ID of the current lure module"` - FirstSeenTimestamp int16 `json:"first_seen_timestamp" doc:"Unix timestamp when the pokestop was first seen"` - SponsorId *int64 `json:"sponsor_id" doc:"Sponsor ID of the pokestop, if sponsored"` - PartnerId *string `json:"partner_id" doc:"Partner ID of the pokestop, if partnered"` - ArScanEligible *int64 `json:"ar_scan_eligible" doc:"Whether the pokestop is eligible for AR scanning"` - PowerUpLevel *int64 `json:"power_up_level" doc:"Power-up level of the pokestop"` - PowerUpPoints *int64 `json:"power_up_points" doc:"Power-up points accumulated for the pokestop"` - PowerUpEndTimestamp *int64 `json:"power_up_end_timestamp" doc:"Unix timestamp when the power-up ends"` - AlternativeQuestType *int64 `json:"alternative_quest_type" doc:"Type of the non-AR quest"` - AlternativeQuestTimestamp *int64 `json:"alternative_quest_timestamp" doc:"Unix timestamp when the non-AR quest was set"` - AlternativeQuestTarget *int64 `json:"alternative_quest_target" doc:"Target count for the non-AR quest"` - AlternativeQuestConditions *string `json:"alternative_quest_conditions" doc:"Serialized conditions of the non-AR quest"` - AlternativeQuestRewards *string `json:"alternative_quest_rewards" doc:"Serialized rewards of the non-AR quest"` - AlternativeQuestTemplate *string `json:"alternative_quest_template" doc:"Template ID of the non-AR quest"` - AlternativeQuestTitle *string `json:"alternative_quest_title" doc:"Title of the non-AR quest"` - AlternativeQuestExpiry *int64 `json:"alternative_quest_expiry" doc:"Unix timestamp when the non-AR quest expires"` - Description *string `json:"description" doc:"Description of the pokestop"` - ShowcaseFocus *string `json:"showcase_focus" doc:"Focus type of the showcase contest"` - ShowcasePokemon *int64 `json:"showcase_pokemon_id" doc:"Pokedex ID of the showcase contest pokemon"` - ShowcasePokemonForm *int64 `json:"showcase_pokemon_form_id" doc:"Form ID of the showcase contest pokemon"` - ShowcasePokemonType *int64 `json:"showcase_pokemon_type_id" doc:"Type ID of the showcase contest pokemon"` - ShowcaseRankingStandard *int64 `json:"showcase_ranking_standard" doc:"Ranking standard of the showcase contest"` - ShowcaseExpiry *int64 `json:"showcase_expiry" doc:"Unix timestamp when the showcase contest expires"` - ShowcaseRankings *string `json:"showcase_rankings" doc:"Serialized showcase contest rankings"` + Id string `json:"id" doc:"Fort ID of the pokestop"` + Lat float64 `json:"lat" doc:"Latitude of the pokestop"` + Lon float64 `json:"lon" doc:"Longitude of the pokestop"` + Name *string `json:"name" doc:"Name of the pokestop"` + Url *string `json:"url" doc:"Image URL of the pokestop"` + LureExpireTimestamp *int64 `json:"lure_expire_timestamp" doc:"Unix timestamp when the current lure expires"` + LastModifiedTimestamp *int64 `json:"last_modified_timestamp" doc:"Unix timestamp when the pokestop was last modified in-game"` + Updated int64 `json:"updated" doc:"Unix timestamp when the record was last updated"` + Enabled *bool `json:"enabled" doc:"Whether the pokestop is enabled"` + QuestType *int64 `json:"quest_type" doc:"Type of the AR quest"` + QuestTimestamp *int64 `json:"quest_timestamp" doc:"Unix timestamp when the AR quest was set"` + QuestTarget *int64 `json:"quest_target" doc:"Target count for the AR quest"` + QuestRewardType *int64 `json:"quest_reward_type" doc:"Reward type of the AR quest (generated from quest_rewards[0].type)"` + QuestItemId *int64 `json:"quest_item_id" doc:"Item id of the AR quest reward, if an item reward"` + QuestRewardAmount *int64 `json:"quest_reward_amount" doc:"Reward amount of the AR quest"` + QuestPokemonId *int64 `json:"quest_pokemon_id" doc:"Pokemon id of the AR quest reward, if a pokemon/candy reward"` + QuestPokemonFormId *int64 `json:"quest_pokemon_form_id" doc:"Form id of the AR quest reward pokemon, if a pokemon reward"` + QuestConditions *string `json:"quest_conditions" doc:"Serialized conditions of the AR quest"` + QuestRewards *json.RawMessage `json:"quest_rewards" doc:"Rewards of the AR quest as native JSON (array of {type, info}); null when no quest"` + QuestTemplate *string `json:"quest_template" doc:"Template ID of the AR quest"` + QuestTitle *string `json:"quest_title" doc:"Title of the AR quest"` + QuestExpiry *int64 `json:"quest_expiry" doc:"Unix timestamp when the AR quest expires"` + CellId *int64 `json:"cell_id" doc:"S2 cell ID the pokestop belongs to"` + Deleted bool `json:"deleted" doc:"Whether the pokestop has been deleted"` + LureId int16 `json:"lure_id" doc:"ID of the current lure module"` + FirstSeenTimestamp int16 `json:"first_seen_timestamp" doc:"Unix timestamp when the pokestop was first seen"` + SponsorId *int64 `json:"sponsor_id" doc:"Sponsor ID of the pokestop, if sponsored"` + PartnerId *string `json:"partner_id" doc:"Partner ID of the pokestop, if partnered"` + ArScanEligible *int64 `json:"ar_scan_eligible" doc:"Whether the pokestop is eligible for AR scanning"` + PowerUpLevel *int64 `json:"power_up_level" doc:"Power-up level of the pokestop"` + PowerUpPoints *int64 `json:"power_up_points" doc:"Power-up points accumulated for the pokestop"` + PowerUpEndTimestamp *int64 `json:"power_up_end_timestamp" doc:"Unix timestamp when the power-up ends"` + AlternativeQuestType *int64 `json:"alternative_quest_type" doc:"Type of the non-AR quest"` + AlternativeQuestTimestamp *int64 `json:"alternative_quest_timestamp" doc:"Unix timestamp when the non-AR quest was set"` + AlternativeQuestTarget *int64 `json:"alternative_quest_target" doc:"Target count for the non-AR quest"` + AlternativeQuestRewardType *int64 `json:"alternative_quest_reward_type" doc:"Reward type of the non-AR quest (generated from alternative_quest_rewards[0].type)"` + AlternativeQuestItemId *int64 `json:"alternative_quest_item_id" doc:"Item id of the non-AR quest reward, if an item reward"` + AlternativeQuestRewardAmount *int64 `json:"alternative_quest_reward_amount" doc:"Reward amount of the non-AR quest"` + AlternativeQuestPokemonId *int64 `json:"alternative_quest_pokemon_id" doc:"Pokemon id of the non-AR quest reward, if a pokemon/candy reward"` + AlternativeQuestPokemonFormId *int64 `json:"alternative_quest_pokemon_form_id" doc:"Form id of the non-AR quest reward pokemon, if a pokemon reward"` + AlternativeQuestConditions *string `json:"alternative_quest_conditions" doc:"Serialized conditions of the non-AR quest"` + AlternativeQuestRewards *json.RawMessage `json:"alternative_quest_rewards" doc:"Rewards of the non-AR quest as native JSON (array of {type, info}); null when no quest"` + AlternativeQuestTemplate *string `json:"alternative_quest_template" doc:"Template ID of the non-AR quest"` + AlternativeQuestTitle *string `json:"alternative_quest_title" doc:"Title of the non-AR quest"` + AlternativeQuestExpiry *int64 `json:"alternative_quest_expiry" doc:"Unix timestamp when the non-AR quest expires"` + Description *string `json:"description" doc:"Description of the pokestop"` + ShowcaseFocus *string `json:"showcase_focus" doc:"Focus type of the showcase contest"` + ShowcasePokemon *int64 `json:"showcase_pokemon_id" doc:"Pokedex ID of the showcase contest pokemon"` + ShowcasePokemonForm *int64 `json:"showcase_pokemon_form_id" doc:"Form ID of the showcase contest pokemon"` + ShowcasePokemonType *int64 `json:"showcase_pokemon_type_id" doc:"Type ID of the showcase contest pokemon"` + ShowcaseRankingStandard *int64 `json:"showcase_ranking_standard" doc:"Ranking standard of the showcase contest"` + ShowcaseExpiry *int64 `json:"showcase_expiry" doc:"Unix timestamp when the showcase contest expires"` + ShowcaseRankings *string `json:"showcase_rankings" doc:"Serialized showcase contest rankings"` + Invasions []ApiPokestopIncident `json:"invasions,omitempty" doc:"Active incidents; present when the pokestop has active incidents (always attempted on by-id, on scans only when with_incidents is set)"` } func buildPokestopResult(stop *Pokestop) ApiPokestopResult { return ApiPokestopResult{ - Id: stop.Id, - Lat: stop.Lat, - Lon: stop.Lon, - Name: stop.Name.Ptr(), - Url: stop.Url.Ptr(), - LureExpireTimestamp: stop.LureExpireTimestamp.Ptr(), - LastModifiedTimestamp: stop.LastModifiedTimestamp.Ptr(), - Updated: stop.Updated, - Enabled: stop.Enabled.Ptr(), - QuestType: stop.QuestType.Ptr(), - QuestTimestamp: stop.QuestTimestamp.Ptr(), - QuestTarget: stop.QuestTarget.Ptr(), - QuestConditions: stop.QuestConditions.Ptr(), - QuestRewards: stop.QuestRewards.Ptr(), - QuestTemplate: stop.QuestTemplate.Ptr(), - QuestTitle: stop.QuestTitle.Ptr(), - QuestExpiry: stop.QuestExpiry.Ptr(), - CellId: stop.CellId.Ptr(), - Deleted: stop.Deleted, - LureId: stop.LureId, - FirstSeenTimestamp: stop.FirstSeenTimestamp, - SponsorId: stop.SponsorId.Ptr(), - PartnerId: stop.PartnerId.Ptr(), - ArScanEligible: stop.ArScanEligible.Ptr(), - PowerUpLevel: stop.PowerUpLevel.Ptr(), - PowerUpPoints: stop.PowerUpPoints.Ptr(), - PowerUpEndTimestamp: stop.PowerUpEndTimestamp.Ptr(), - AlternativeQuestType: stop.AlternativeQuestType.Ptr(), - AlternativeQuestTimestamp: stop.AlternativeQuestTimestamp.Ptr(), - AlternativeQuestTarget: stop.AlternativeQuestTarget.Ptr(), - AlternativeQuestConditions: stop.AlternativeQuestConditions.Ptr(), - AlternativeQuestRewards: stop.AlternativeQuestRewards.Ptr(), - AlternativeQuestTemplate: stop.AlternativeQuestTemplate.Ptr(), - AlternativeQuestTitle: stop.AlternativeQuestTitle.Ptr(), - AlternativeQuestExpiry: stop.AlternativeQuestExpiry.Ptr(), - Description: stop.Description.Ptr(), - ShowcaseFocus: stop.ShowcaseFocus.Ptr(), - ShowcasePokemon: stop.ShowcasePokemon.Ptr(), - ShowcasePokemonForm: stop.ShowcasePokemonForm.Ptr(), - ShowcasePokemonType: stop.ShowcasePokemonType.Ptr(), - ShowcaseRankingStandard: stop.ShowcaseRankingStandard.Ptr(), - ShowcaseExpiry: stop.ShowcaseExpiry.Ptr(), - ShowcaseRankings: stop.ShowcaseRankings.Ptr(), + Id: stop.Id, + Lat: stop.Lat, + Lon: stop.Lon, + Name: stop.Name.Ptr(), + Url: stop.Url.Ptr(), + LureExpireTimestamp: stop.LureExpireTimestamp.Ptr(), + LastModifiedTimestamp: stop.LastModifiedTimestamp.Ptr(), + Updated: stop.Updated, + Enabled: stop.Enabled.Ptr(), + QuestType: stop.QuestType.Ptr(), + QuestTimestamp: stop.QuestTimestamp.Ptr(), + QuestTarget: stop.QuestTarget.Ptr(), + QuestRewardType: stop.QuestRewardType.Ptr(), + QuestItemId: stop.QuestItemId.Ptr(), + QuestRewardAmount: stop.QuestRewardAmount.Ptr(), + QuestPokemonId: stop.QuestPokemonId.Ptr(), + QuestPokemonFormId: stop.QuestPokemonFormId.Ptr(), + QuestConditions: stop.QuestConditions.Ptr(), + QuestRewards: jsonRaw(stop.QuestRewards), + QuestTemplate: stop.QuestTemplate.Ptr(), + QuestTitle: stop.QuestTitle.Ptr(), + QuestExpiry: stop.QuestExpiry.Ptr(), + CellId: stop.CellId.Ptr(), + Deleted: stop.Deleted, + LureId: stop.LureId, + FirstSeenTimestamp: stop.FirstSeenTimestamp, + SponsorId: stop.SponsorId.Ptr(), + PartnerId: stop.PartnerId.Ptr(), + ArScanEligible: stop.ArScanEligible.Ptr(), + PowerUpLevel: stop.PowerUpLevel.Ptr(), + PowerUpPoints: stop.PowerUpPoints.Ptr(), + PowerUpEndTimestamp: stop.PowerUpEndTimestamp.Ptr(), + AlternativeQuestType: stop.AlternativeQuestType.Ptr(), + AlternativeQuestTimestamp: stop.AlternativeQuestTimestamp.Ptr(), + AlternativeQuestTarget: stop.AlternativeQuestTarget.Ptr(), + AlternativeQuestRewardType: stop.AlternativeQuestRewardType.Ptr(), + AlternativeQuestItemId: stop.AlternativeQuestItemId.Ptr(), + AlternativeQuestRewardAmount: stop.AlternativeQuestRewardAmount.Ptr(), + AlternativeQuestPokemonId: stop.AlternativeQuestPokemonId.Ptr(), + AlternativeQuestPokemonFormId: stop.AlternativeQuestPokemonFormId.Ptr(), + AlternativeQuestConditions: stop.AlternativeQuestConditions.Ptr(), + AlternativeQuestRewards: jsonRaw(stop.AlternativeQuestRewards), + AlternativeQuestTemplate: stop.AlternativeQuestTemplate.Ptr(), + AlternativeQuestTitle: stop.AlternativeQuestTitle.Ptr(), + AlternativeQuestExpiry: stop.AlternativeQuestExpiry.Ptr(), + Description: stop.Description.Ptr(), + ShowcaseFocus: stop.ShowcaseFocus.Ptr(), + ShowcasePokemon: stop.ShowcasePokemon.Ptr(), + ShowcasePokemonForm: stop.ShowcasePokemonForm.Ptr(), + ShowcasePokemonType: stop.ShowcasePokemonType.Ptr(), + ShowcaseRankingStandard: stop.ShowcaseRankingStandard.Ptr(), + ShowcaseExpiry: stop.ShowcaseExpiry.Ptr(), + ShowcaseRankings: stop.ShowcaseRankings.Ptr(), } } func BuildPokestopResult(stop *Pokestop) ApiPokestopResult { return buildPokestopResult(stop) } + +// ApiPokestopIncident is one active incident (whole row) on a pokestop, as +// returned in a scan/by-id response when with_incidents is set. Sourced from +// incidentCache via the FortLookup fetch handle; nullable slots are pointers. +type ApiPokestopIncident struct { + Id string `json:"id" doc:"Incident id"` + PokestopId string `json:"pokestop_id" doc:"Fort id of the parent pokestop"` + DisplayType int16 `json:"display_type" doc:"Incident display type (1-4 rocket, 7 goldstop, 8 kecleon, 9 showcase)"` + Style int16 `json:"style" doc:"Incident style"` + Character int16 `json:"character" doc:"Invasion character id (grunt/leader/giovanni); 0 for non-rocket"` + StartTime int64 `json:"start" doc:"Unix timestamp when the incident started"` + ExpirationTime int64 `json:"expiration" doc:"Unix timestamp when the incident expires"` + Confirmed bool `json:"confirmed" doc:"True when the lineup is confirmed (grunts only)"` + Updated int64 `json:"updated" doc:"Unix timestamp when the incident was last updated"` + Slot1PokemonId *int64 `json:"slot_1_pokemon_id" doc:"Confirmed lead pokemon id, else null"` + Slot1Form *int64 `json:"slot_1_form" doc:"Confirmed lead pokemon form, else null"` + Slot2PokemonId *int64 `json:"slot_2_pokemon_id" doc:"Slot 2 pokemon id, else null"` + Slot2Form *int64 `json:"slot_2_form" doc:"Slot 2 form, else null"` + Slot3PokemonId *int64 `json:"slot_3_pokemon_id" doc:"Slot 3 pokemon id, else null"` + Slot3Form *int64 `json:"slot_3_form" doc:"Slot 3 form, else null"` +} + +func buildPokestopIncident(inc *Incident) ApiPokestopIncident { + return ApiPokestopIncident{ + Id: inc.Id, + PokestopId: inc.PokestopId, + DisplayType: inc.DisplayType, + Style: inc.Style, + Character: inc.Character, + StartTime: inc.StartTime, + ExpirationTime: inc.ExpirationTime, + Confirmed: inc.Confirmed, + Updated: inc.Updated, + Slot1PokemonId: inc.Slot1PokemonId.Ptr(), + Slot1Form: inc.Slot1Form.Ptr(), + Slot2PokemonId: inc.Slot2PokemonId.Ptr(), + Slot2Form: inc.Slot2Form.Ptr(), + Slot3PokemonId: inc.Slot3PokemonId.Ptr(), + Slot3Form: inc.Slot3Form.Ptr(), + } +} + +// CollectPokestopIncidents returns the whole-row active incidents for a fort, +// resolved from incidentCache via the string handles in the fort's FortLookup +// (read-through to DB on the rare cache miss). Callers MUST NOT hold the +// pokestop lock — this locks incidents, and saveIncidentRecord locks +// incident->pokestop, so holding pokestop here would invert the order. +func CollectPokestopIncidents(ctx context.Context, dbDetails db.DbDetails, fortId string, now int64) []ApiPokestopIncident { + fl, ok := fortLookupCache.Load(fortId) + if !ok || len(fl.Incidents) == 0 { + return nil + } + out := make([]ApiPokestopIncident, 0, len(fl.Incidents)) + for _, li := range fl.Incidents { + if li.ExpireTimestamp <= now || li.Id == "" { + continue + } + inc, unlock, err := getIncidentRecordReadOnly(ctx, dbDetails, li.Id, "API.CollectPokestopIncidents") + if err != nil || inc == nil { + if unlock != nil { + unlock() + } + continue + } + out = append(out, buildPokestopIncident(inc)) + unlock() + } + return out +} diff --git a/decoder/api_pokestop_incidents_test.go b/decoder/api_pokestop_incidents_test.go new file mode 100644 index 00000000..0fe07abb --- /dev/null +++ b/decoder/api_pokestop_incidents_test.go @@ -0,0 +1,50 @@ +package decoder + +import ( + "context" + "testing" + "time" + + db "golbat/db" + ottercache "golbat/ottercache" + + "github.com/guregu/null/v6" + "github.com/puzpuzpuz/xsync/v4" +) + +func newTestIncidentCache() *ottercache.OtterCache[string, *Incident] { + return ottercache.NewOtterCache(ottercache.OtterCacheConfig[string, *Incident]{ + Name: "incident-test", DefaultTTL: 60 * time.Minute, + }) +} + +// CollectPokestopIncidents returns the whole active-incident rows for a fort, +// looked up from incidentCache via the FortLookup handles, skipping expired. +func TestCollectPokestopIncidents(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + incidentCache = newTestIncidentCache() + now := int64(1_000_000) + + active := &Incident{IncidentData: IncidentData{ + Id: "inc-active", PokestopId: "s1", DisplayType: 1, Character: 5, + Confirmed: true, Slot1PokemonId: null.IntFrom(41), ExpirationTime: now + 100, + }} + expired := &Incident{IncidentData: IncidentData{ + Id: "inc-expired", PokestopId: "s1", DisplayType: 3, Character: 30, ExpirationTime: now - 1, + }} + incidentCache.Set("inc-active", active, 0) + incidentCache.Set("inc-expired", expired, 0) + + fortLookupCache.Store("s1", FortLookup{FortType: POKESTOP, Incidents: []FortLookupIncident{ + {Id: "inc-active", DisplayType: 1, Character: 5, ExpireTimestamp: now + 100}, + {Id: "inc-expired", DisplayType: 3, Character: 30, ExpireTimestamp: now - 1}, + }}) + + got := CollectPokestopIncidents(context.Background(), db.DbDetails{}, "s1", now) + if len(got) != 1 { + t.Fatalf("expected 1 active incident, got %d: %+v", len(got), got) + } + if got[0].Id != "inc-active" || got[0].Character != 5 || got[0].Slot1PokemonId == nil || *got[0].Slot1PokemonId != 41 { + t.Fatalf("wrong incident payload: %+v", got[0]) + } +} diff --git a/decoder/api_pokestop_test.go b/decoder/api_pokestop_test.go index 84b479fc..241d9754 100644 --- a/decoder/api_pokestop_test.go +++ b/decoder/api_pokestop_test.go @@ -25,9 +25,12 @@ func goldenSnapshotPokestop() *Pokestop { QuestType: null.IntFrom(7), QuestTimestamp: null.IntFrom(1699991000), QuestTarget: null.IntFrom(3), - QuestConditions: null.StringFrom("[]"), - QuestRewards: null.StringFrom("[{\"type\":1}]"), - QuestTemplate: null.StringFrom("challenge_template"), + QuestRewardType: null.IntFrom(1), + QuestRewardAmount: null.IntFrom(100), + // QuestItemId, QuestPokemonId, QuestPokemonFormId left null (xp reward) + QuestConditions: null.StringFrom("[]"), + QuestRewards: null.StringFrom("[{\"type\":1}]"), + QuestTemplate: null.StringFrom("challenge_template"), // QuestTitle intentionally left null QuestExpiry: null.IntFrom(1700003600), CellId: null.IntFrom(1234567890123), @@ -36,14 +39,18 @@ func goldenSnapshotPokestop() *Pokestop { // FirstSeenTimestamp is int16, plain field FirstSeenTimestamp: 0, // SponsorId intentionally left null - PartnerId: null.StringFrom("partner-1"), - ArScanEligible: null.IntFrom(1), - PowerUpLevel: null.IntFrom(2), - PowerUpPoints: null.IntFrom(50), - PowerUpEndTimestamp: null.IntFrom(1700007200), - AlternativeQuestType: null.IntFrom(7), - AlternativeQuestTimestamp: null.IntFrom(1699992000), - AlternativeQuestTarget: null.IntFrom(5), + PartnerId: null.StringFrom("partner-1"), + ArScanEligible: null.IntFrom(1), + PowerUpLevel: null.IntFrom(2), + PowerUpPoints: null.IntFrom(50), + PowerUpEndTimestamp: null.IntFrom(1700007200), + AlternativeQuestType: null.IntFrom(7), + AlternativeQuestTimestamp: null.IntFrom(1699992000), + AlternativeQuestTarget: null.IntFrom(5), + AlternativeQuestRewardType: null.IntFrom(2), + AlternativeQuestItemId: null.IntFrom(1), + AlternativeQuestRewardAmount: null.IntFrom(3), + // AlternativeQuestPokemonId, AlternativeQuestPokemonFormId left null (item reward) // AlternativeQuestConditions intentionally left null AlternativeQuestRewards: null.StringFrom("[{\"type\":2}]"), AlternativeQuestTemplate: null.StringFrom("alt_template"), @@ -71,7 +78,7 @@ func TestBuildPokestopResult_GoldenSnapshot(t *testing.T) { t.Fatalf("marshal: %v", err) } - const want = `{"id":"stop-abc","lat":12.3456,"lon":-65.4321,"name":"Test Pokestop","url":"https://example.com/stop.png","lure_expire_timestamp":null,"last_modified_timestamp":1699990000,"updated":1699999999,"enabled":true,"quest_type":7,"quest_timestamp":1699991000,"quest_target":3,"quest_conditions":"[]","quest_rewards":"[{\"type\":1}]","quest_template":"challenge_template","quest_title":null,"quest_expiry":1700003600,"cell_id":1234567890123,"deleted":false,"lure_id":501,"first_seen_timestamp":0,"sponsor_id":null,"partner_id":"partner-1","ar_scan_eligible":1,"power_up_level":2,"power_up_points":50,"power_up_end_timestamp":1700007200,"alternative_quest_type":7,"alternative_quest_timestamp":1699992000,"alternative_quest_target":5,"alternative_quest_conditions":null,"alternative_quest_rewards":"[{\"type\":2}]","alternative_quest_template":"alt_template","alternative_quest_title":"Alt Quest","alternative_quest_expiry":1700003601,"description":"A test pokestop","showcase_focus":null,"showcase_pokemon_id":150,"showcase_pokemon_form_id":0,"showcase_pokemon_type_id":1,"showcase_ranking_standard":0,"showcase_expiry":null,"showcase_rankings":"[]"}` + const want = `{"id":"stop-abc","lat":12.3456,"lon":-65.4321,"name":"Test Pokestop","url":"https://example.com/stop.png","lure_expire_timestamp":null,"last_modified_timestamp":1699990000,"updated":1699999999,"enabled":true,"quest_type":7,"quest_timestamp":1699991000,"quest_target":3,"quest_reward_type":1,"quest_item_id":null,"quest_reward_amount":100,"quest_pokemon_id":null,"quest_pokemon_form_id":null,"quest_conditions":"[]","quest_rewards":[{"type":1}],"quest_template":"challenge_template","quest_title":null,"quest_expiry":1700003600,"cell_id":1234567890123,"deleted":false,"lure_id":501,"first_seen_timestamp":0,"sponsor_id":null,"partner_id":"partner-1","ar_scan_eligible":1,"power_up_level":2,"power_up_points":50,"power_up_end_timestamp":1700007200,"alternative_quest_type":7,"alternative_quest_timestamp":1699992000,"alternative_quest_target":5,"alternative_quest_reward_type":2,"alternative_quest_item_id":1,"alternative_quest_reward_amount":3,"alternative_quest_pokemon_id":null,"alternative_quest_pokemon_form_id":null,"alternative_quest_conditions":null,"alternative_quest_rewards":[{"type":2}],"alternative_quest_template":"alt_template","alternative_quest_title":"Alt Quest","alternative_quest_expiry":1700003601,"description":"A test pokestop","showcase_focus":null,"showcase_pokemon_id":150,"showcase_pokemon_form_id":0,"showcase_pokemon_type_id":1,"showcase_ranking_standard":0,"showcase_expiry":null,"showcase_rankings":"[]"}` if string(got) != want { t.Errorf("wire format changed.\n got: %s\nwant: %s", got, want) diff --git a/decoder/api_station.go b/decoder/api_station.go index 2ded3695..41496a02 100644 --- a/decoder/api_station.go +++ b/decoder/api_station.go @@ -8,29 +8,34 @@ import ( // columns are represented as pointers (nil => JSON null) without omitempty so // every key is always present. type ApiStationResult struct { - Id string `json:"id" doc:"Station ID"` - Lat float64 `json:"lat" doc:"Latitude of the station"` - Lon float64 `json:"lon" doc:"Longitude of the station"` - Name string `json:"name" doc:"Name of the station"` - StartTime int64 `json:"start_time" doc:"Unix timestamp when the station becomes active"` - EndTime int64 `json:"end_time" doc:"Unix timestamp when the station expires"` - IsBattleAvailable bool `json:"is_battle_available" doc:"Whether a battle is currently available at the station"` - Updated int64 `json:"updated" doc:"Unix timestamp when the record was last updated"` - BattleLevel *int64 `json:"battle_level" doc:"Level of the current (top) battle"` - BattleStart *int64 `json:"battle_start" doc:"Unix timestamp when the current battle starts"` - BattleEnd *int64 `json:"battle_end" doc:"Unix timestamp when the current battle ends"` - BattlePokemonId *int64 `json:"battle_pokemon_id" doc:"Pokedex ID of the battle pokemon"` - BattlePokemonForm *int64 `json:"battle_pokemon_form" doc:"Form ID of the battle pokemon"` - BattlePokemonCostume *int64 `json:"battle_pokemon_costume" doc:"Costume ID of the battle pokemon"` - BattlePokemonGender *int64 `json:"battle_pokemon_gender" doc:"Gender of the battle pokemon"` - BattlePokemonAlignment *int64 `json:"battle_pokemon_alignment" doc:"Alignment of the battle pokemon"` - BattlePokemonBreadMode *int64 `json:"battle_pokemon_bread_mode" doc:"Bread mode of the battle pokemon"` - BattlePokemonMove1 *int64 `json:"battle_pokemon_move_1" doc:"First move ID of the battle pokemon"` - BattlePokemonMove2 *int64 `json:"battle_pokemon_move_2" doc:"Second move ID of the battle pokemon"` - TotalStationedPokemon *int64 `json:"total_stationed_pokemon" doc:"Total number of pokemon stationed"` - TotalStationedGmax *int64 `json:"total_stationed_gmax" doc:"Total number of Gigantamax pokemon stationed"` - StationedPokemon *string `json:"stationed_pokemon" doc:"Serialized list of stationed pokemon"` - Battles []ApiStationBattleResult `json:"battles,omitempty" doc:"Known battles at this station"` + Id string `json:"id" doc:"Station ID"` + Lat float64 `json:"lat" doc:"Latitude of the station"` + Lon float64 `json:"lon" doc:"Longitude of the station"` + Name string `json:"name" doc:"Name of the station"` + CellId int64 `json:"cell_id" doc:"S2 cell ID the station belongs to"` + StartTime int64 `json:"start_time" doc:"Unix timestamp when the station becomes active"` + EndTime int64 `json:"end_time" doc:"Unix timestamp when the station expires"` + CooldownComplete int64 `json:"cooldown_complete" doc:"Unix timestamp when the station cooldown completes"` + IsBattleAvailable bool `json:"is_battle_available" doc:"Whether a battle is currently available at the station"` + IsInactive bool `json:"is_inactive" doc:"Whether the station is inactive"` + Updated int64 `json:"updated" doc:"Unix timestamp when the record was last updated"` + BattleLevel *int64 `json:"battle_level" doc:"Level of the current (top) battle"` + BattleStart *int64 `json:"battle_start" doc:"Unix timestamp when the current battle starts"` + BattleEnd *int64 `json:"battle_end" doc:"Unix timestamp when the current battle ends"` + BattlePokemonId *int64 `json:"battle_pokemon_id" doc:"Pokedex ID of the battle pokemon"` + BattlePokemonForm *int64 `json:"battle_pokemon_form" doc:"Form ID of the battle pokemon"` + BattlePokemonCostume *int64 `json:"battle_pokemon_costume" doc:"Costume ID of the battle pokemon"` + BattlePokemonGender *int64 `json:"battle_pokemon_gender" doc:"Gender of the battle pokemon"` + BattlePokemonAlignment *int64 `json:"battle_pokemon_alignment" doc:"Alignment of the battle pokemon"` + BattlePokemonBreadMode *int64 `json:"battle_pokemon_bread_mode" doc:"Bread mode of the battle pokemon"` + BattlePokemonMove1 *int64 `json:"battle_pokemon_move_1" doc:"First move ID of the battle pokemon"` + BattlePokemonMove2 *int64 `json:"battle_pokemon_move_2" doc:"Second move ID of the battle pokemon"` + BattlePokemonStamina *int64 `json:"battle_pokemon_stamina" doc:"Stamina of the top battle pokemon"` + BattlePokemonCpMultiplier *float64 `json:"battle_pokemon_cp_multiplier" doc:"CP multiplier of the top battle pokemon"` + TotalStationedPokemon *int64 `json:"total_stationed_pokemon" doc:"Total number of pokemon stationed"` + TotalStationedGmax *int64 `json:"total_stationed_gmax" doc:"Total number of Gigantamax pokemon stationed"` + StationedPokemon *string `json:"stationed_pokemon" doc:"Serialized list of stationed pokemon"` + Battles []ApiStationBattleResult `json:"battles,omitempty" doc:"Known battles at this station"` } // ApiStationBattleResult is one battle entry for a station. @@ -60,9 +65,12 @@ func BuildStationResult(station *Station) ApiStationResult { Lat: station.Lat, Lon: station.Lon, Name: station.Name, + CellId: station.CellId, StartTime: station.StartTime, EndTime: station.EndTime, + CooldownComplete: station.CooldownComplete, IsBattleAvailable: station.IsBattleAvailable, + IsInactive: station.IsInactive, Updated: station.Updated, TotalStationedPokemon: station.TotalStationedPokemon.Ptr(), TotalStationedGmax: station.TotalStationedGmax.Ptr(), diff --git a/decoder/api_station_available.go b/decoder/api_station_available.go new file mode 100644 index 00000000..802212b0 --- /dev/null +++ b/decoder/api_station_available.go @@ -0,0 +1,68 @@ +package decoder + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +// ApiStationBattleAvailable is one distinct active (battle_level, pokemon, form) +// option on resident stations. ReactMap derives its -
and j keys. +type ApiStationBattleAvailable struct { + BattleLevel int8 `json:"battle_level" doc:"Max battle level"` + PokemonId int16 `json:"pokemon_id" doc:"Battle pokemon id, else 0"` + Form int16 `json:"form" doc:"Battle pokemon form id, else 0"` + Count int `json:"count" doc:"Number of resident stations with this active battle option"` +} + +// ApiAvailableStations is the whole-instance station filter snapshot served by +// GET /api/station/available. +type ApiAvailableStations struct { + Battles []ApiStationBattleAvailable `json:"battles" doc:"Distinct active battle level/pokemon options on resident stations"` +} + +// GetAvailableStations builds the station filter snapshot from a single +// fortLookupCache range. Mirrors isFortDnfMatch's station branch: iterate the +// StationBattles slice when present, else fall back to the top-battle +// projection; skip expired and level-0 battles. +// Unlike isFortDnfMatch, level-0 battles are excluded here (ReactMap's !battle_level convention). +func GetAvailableStations(now int64) *ApiAvailableStations { + start := time.Now() + res := &ApiAvailableStations{Battles: []ApiStationBattleAvailable{}} + battles := map[ApiStationBattleAvailable]int{} + forts := 0 + + add := func(level int8, pokemonId, form int16, end int64) { + if level == 0 || end <= now { + return + } + battles[ApiStationBattleAvailable{BattleLevel: level, PokemonId: pokemonId, Form: form}]++ + } + + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + if fl.FortType != STATION { + return true + } + forts++ + if len(fl.StationBattles) == 0 { + add(fl.BattleLevel, fl.BattlePokemonId, fl.BattlePokemonForm, fl.BattleEndTimestamp) + return true + } + for _, b := range fl.StationBattles { + add(b.BattleLevel, b.BattlePokemonId, b.BattlePokemonForm, b.BattleEndTimestamp) + } + return true + }) + + for k, n := range battles { + k.Count = n + res.Battles = append(res.Battles, k) + } + + if statsCollector != nil { + statsCollector.ObserveApiScan("available-stations", time.Since(start).Seconds()) + } + log.Infof("available-stations built in %s: scanned %d stations -> %d battle options", + time.Since(start), forts, len(res.Battles)) + return res +} diff --git a/decoder/api_station_available_test.go b/decoder/api_station_available_test.go new file mode 100644 index 00000000..4b0f0e97 --- /dev/null +++ b/decoder/api_station_available_test.go @@ -0,0 +1,39 @@ +package decoder + +import ( + "testing" + + "github.com/puzpuzpuz/xsync/v4" +) + +func TestGetAvailableStations(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + now := int64(1_000_000) + + // station with two active battles (multi-battle path) + one expired + fortLookupCache.Store("st1", FortLookup{FortType: STATION, StationBattles: []FortLookupStationBattle{ + {BattleLevel: 3, BattlePokemonId: 150, BattlePokemonForm: 0, BattleEndTimestamp: now + 100}, + {BattleLevel: 5, BattlePokemonId: 384, BattlePokemonForm: 0, BattleEndTimestamp: now + 100}, + {BattleLevel: 1, BattlePokemonId: 1, BattleEndTimestamp: now - 1}, // expired -> excluded + }}) + // station with only the top-battle projection (no StationBattles slice) + fortLookupCache.Store("st2", FortLookup{FortType: STATION, + BattleLevel: 6, BattlePokemonId: 999, BattlePokemonForm: 0, BattleEndTimestamp: now + 100, + }) + // station with a level-0 battle -> excluded + fortLookupCache.Store("st3", FortLookup{FortType: STATION, StationBattles: []FortLookupStationBattle{ + {BattleLevel: 0, BattlePokemonId: 5, BattleEndTimestamp: now + 100}, + }}) + fortLookupCache.Store("g1", FortLookup{FortType: GYM, TeamId: 1}) // ignored + + res := GetAvailableStations(now) + // expect: (3,150),(5,384) from st1, (6,999) from st2 = 3 distinct; expired + level-0 excluded + if len(res.Battles) != 3 { + t.Fatalf("battles: %+v", res.Battles) + } + for _, b := range res.Battles { + if b.BattleLevel == 0 || b.PokemonId == 1 { + t.Fatalf("excluded battle leaked: %+v", b) + } + } +} diff --git a/decoder/api_station_test.go b/decoder/api_station_test.go index 31133076..9d86da20 100644 --- a/decoder/api_station_test.go +++ b/decoder/api_station_test.go @@ -17,11 +17,14 @@ func goldenSnapshotStation() *Station { Lat: 45.6789, Lon: -120.9876, Name: "Test Station", + CellId: 1234567890123, StartTime: 1699990000, EndTime: 1700003600, + CooldownComplete: 1700002000, IsBattleAvailable: true, - Updated: 1699999999, - BattleLevel: null.IntFrom(5), + // IsInactive intentionally left false + Updated: 1699999999, + BattleLevel: null.IntFrom(5), // BattleStart intentionally left null BattleEnd: null.IntFrom(1700001000), BattlePokemonId: null.IntFrom(150), @@ -51,7 +54,7 @@ func TestBuildStationResult_GoldenSnapshot(t *testing.T) { // Battle flat-fields are now projected from the station battle cache // (getKnownStationBattles), which is empty here, so they serialize as null. - const want = `{"id":"station-abc","lat":45.6789,"lon":-120.9876,"name":"Test Station","start_time":1699990000,"end_time":1700003600,"is_battle_available":true,"updated":1699999999,"battle_level":null,"battle_start":null,"battle_end":null,"battle_pokemon_id":null,"battle_pokemon_form":null,"battle_pokemon_costume":null,"battle_pokemon_gender":null,"battle_pokemon_alignment":null,"battle_pokemon_bread_mode":null,"battle_pokemon_move_1":null,"battle_pokemon_move_2":null,"total_stationed_pokemon":6,"total_stationed_gmax":null,"stationed_pokemon":"[{\"pokemon_id\":150}]"}` + const want = `{"id":"station-abc","lat":45.6789,"lon":-120.9876,"name":"Test Station","cell_id":1234567890123,"start_time":1699990000,"end_time":1700003600,"cooldown_complete":1700002000,"is_battle_available":true,"is_inactive":false,"updated":1699999999,"battle_level":null,"battle_start":null,"battle_end":null,"battle_pokemon_id":null,"battle_pokemon_form":null,"battle_pokemon_costume":null,"battle_pokemon_gender":null,"battle_pokemon_alignment":null,"battle_pokemon_bread_mode":null,"battle_pokemon_move_1":null,"battle_pokemon_move_2":null,"battle_pokemon_stamina":null,"battle_pokemon_cp_multiplier":null,"total_stationed_pokemon":6,"total_stationed_gmax":null,"stationed_pokemon":"[{\"pokemon_id\":150}]"}` if string(got) != want { t.Errorf("wire format changed.\n got: %s\nwant: %s", got, want) diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 07dc76d8..7e02ba55 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -58,11 +58,13 @@ type FortLookup struct { ShowcaseExpiry int64 // used to check expiry at filter time // Station - BattleEndTimestamp int64 // used to check expiry at filter time - BattleLevel int8 - BattlePokemonId int16 - BattlePokemonForm int16 - StationBattles []FortLookupStationBattle + StationEndTimestamp int64 // station end_time; liveness gate at filter time + BattleEndTimestamp int64 // used to check expiry at filter time + BattleLevel int8 + BattlePokemonId int16 + BattlePokemonForm int16 + StationBattles []FortLookupStationBattle + TotalStationedGmax int16 } var fortLookupCache *xsync.Map[string, FortLookup] @@ -255,10 +257,12 @@ func updateStationLookup(station *Station) { func updateStationLookupWithBattles(station *Station, stationBattles []StationBattleData) { battles := buildFortLookupStationBattlesFromSlice(stationBattles) lookup := FortLookup{ - FortType: STATION, - Lat: station.Lat, - Lon: station.Lon, - StationBattles: battles, + FortType: STATION, + Lat: station.Lat, + Lon: station.Lon, + StationBattles: battles, + TotalStationedGmax: int16(station.TotalStationedGmax.ValueOrZero()), + StationEndTimestamp: station.EndTime, } applyTopStationBattleToFortLookup(&lookup, stationBattles) fortLookupCache.Store(station.Id, lookup) @@ -270,6 +274,7 @@ func updateStationLookupWithBattles(station *Station, stationBattles []StationBa func updatePokestopIncidentLookup(pokestopId string, incident *Incident) { now := time.Now().Unix() updated := FortLookupIncident{ + Id: incident.Id, DisplayType: int8(incident.DisplayType), Style: int8(incident.Style), Character: incident.Character, diff --git a/decoder/fort_incident_id_test.go b/decoder/fort_incident_id_test.go new file mode 100644 index 00000000..b5d5ee55 --- /dev/null +++ b/decoder/fort_incident_id_test.go @@ -0,0 +1,34 @@ +package decoder + +import ( + "testing" + + "github.com/guregu/null/v6" + "github.com/puzpuzpuz/xsync/v4" +) + +// updatePokestopIncidentLookup must carry the incident Id onto the FortLookup +// projection so the scan can fetch the whole incident row from incidentCache. +func TestUpdatePokestopIncidentLookupCarriesId(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + const id = "stop-1" + fortLookupCache.Store(id, FortLookup{FortType: POKESTOP, Lat: 1, Lon: 2}) + + inc := &Incident{IncidentData: IncidentData{ + Id: "-1016089077232382347", + DisplayType: 1, + Character: 5, + Confirmed: true, + Slot1PokemonId: null.IntFrom(41), + ExpirationTime: 9_999_999_999, + }} + updatePokestopIncidentLookup(id, inc) + + fl, ok := fortLookupCache.Load(id) + if !ok || len(fl.Incidents) != 1 { + t.Fatalf("expected 1 incident, got %+v", fl.Incidents) + } + if fl.Incidents[0].Id != "-1016089077232382347" { + t.Fatalf("incident Id not carried: %q", fl.Incidents[0].Id) + } +} diff --git a/decoder/preload.go b/decoder/preload.go index 586331bf..97e12453 100644 --- a/decoder/preload.go +++ b/decoder/preload.go @@ -64,9 +64,10 @@ func PreloadForts(dbDetails db.DbDetails, populateRtree bool) error { startTime := time.Now() var wg sync.WaitGroup - var pokestopCount, gymCount int32 + var pokestopCount, gymCount, stationCount int32 - wg.Add(2) + // Phase 1: forts (pokestops, gyms, stations) in parallel. + wg.Add(3) go func() { defer wg.Done() pokestopCount = preloadPokestops(dbDetails, populateRtree) @@ -75,10 +76,17 @@ func PreloadForts(dbDetails db.DbDetails, populateRtree bool) error { defer wg.Done() gymCount = preloadGyms(dbDetails, populateRtree) }() + go func() { + defer wg.Done() + stationCount = preloadStations(dbDetails, populateRtree) + }() wg.Wait() - log.Infof("PreloadForts: loaded %d pokestops and %d gyms in %v (rtree=%v)", - pokestopCount, gymCount, time.Since(startTime), populateRtree) + // Phase 2: station battles depend on stationCache being populated. + stationBattleCount := preloadStationBattles(dbDetails, populateRtree) + + log.Infof("PreloadForts: loaded %d pokestops, %d gyms, %d stations, %d station battles in %v (rtree=%v)", + pokestopCount, gymCount, stationCount, stationBattleCount, time.Since(startTime), populateRtree) return nil } diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 0b4ed04d..d3f03cf0 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -49,6 +49,7 @@ type FortLookupStationBattle struct { // incidents on a stop so concurrent incidents (e.g. an invasion + a showcase) don't // clobber one another. type FortLookupIncident struct { + Id string // incident id — fetch handle into incidentCache (not DNF-used) DisplayType int8 Style int8 Character int16 @@ -445,6 +446,8 @@ func applyTopStationBattleToApiStationResult(result *ApiStationResult, battles [ result.BattlePokemonBreadMode = battle.BattlePokemonBreadMode.Ptr() result.BattlePokemonMove1 = battle.BattlePokemonMove1.Ptr() result.BattlePokemonMove2 = battle.BattlePokemonMove2.Ptr() + result.BattlePokemonStamina = battle.BattlePokemonStamina.Ptr() + result.BattlePokemonCpMultiplier = battle.BattlePokemonCpMultiplier.Ptr() } func applyTopStationBattleToStationWebhook(hook *StationWebhook, battles []StationBattleData) { diff --git a/huma_routes_test.go b/huma_routes_test.go index c3360233..7613bfba 100644 --- a/huma_routes_test.go +++ b/huma_routes_test.go @@ -246,7 +246,7 @@ func TestTier3ReadEndpoints(t *testing.T) { }) } -// TestTier3RoutesRegisterInSpec asserts all seven tier-3 operations appear in +// TestTier3RoutesRegisterInSpec asserts all eight tier-3 operations appear in // the OpenAPI spec at their expected method+path (registration smoke test for // the endpoints that need a DB and so are not exercised end-to-end here). func TestTier3RoutesRegisterInSpec(t *testing.T) { @@ -271,6 +271,7 @@ func TestTier3RoutesRegisterInSpec(t *testing.T) { {"post", "/api/station/query"}, {"post", "/api/gym/search"}, {"get", "/api/gym/id/{gym_id}"}, + {"get", "/api/station/id/{station_id}"}, {"get", "/api/pokestop/id/{fort_id}"}, {"get", "/api/tappable/id/{tappable_id}"}, {"post", "/api/pokestop-positions"}, @@ -503,3 +504,74 @@ func TestTier4OperationalEndpoints(t *testing.T) { } }) } + +// TestHumaStationByIdRoute verifies the station by-id route is registered and +// requires the secret; the 404-for-unknown-id path needs a DB, so it is covered +// by the registration smoke test (TestTier3RoutesRegisterInSpec) instead. +func TestHumaStationByIdRoute(t *testing.T) { + prev := config.Config.ApiSecret + config.Config.ApiSecret = "topsecret" + defer func() { config.Config.ApiSecret = prev }() + + _, api := humatest.New(t, newHumaConfig("test")) + api.UseMiddleware(golbatSecretMiddleware(api)) + registerTier3Routes(api) + + t.Run("no secret is 401", func(t *testing.T) { + resp := api.Get("/api/station/id/does-not-exist") + if resp.Code != http.StatusUnauthorized { + t.Errorf("got %d, want 401", resp.Code) + } + }) +} + +func TestHumaGymAvailableRoute(t *testing.T) { + prevSecret := config.Config.ApiSecret + prevFim := config.Config.FortInMemory + config.Config.ApiSecret = "topsecret" + defer func() { config.Config.ApiSecret = prevSecret; config.Config.FortInMemory = prevFim }() + + _, api := humatest.New(t, newHumaConfig("test")) + api.UseMiddleware(golbatSecretMiddleware(api)) + registerFortScanRoutes(api) + + config.Config.FortInMemory = false + if resp := api.Get("/api/gym/available", "X-Golbat-Secret: topsecret"); resp.Code != http.StatusServiceUnavailable { + t.Errorf("fim off: got %d, want 503", resp.Code) + } + config.Config.FortInMemory = true + resp := api.Get("/api/gym/available", "X-Golbat-Secret: topsecret") + if resp.Code != http.StatusOK { + t.Fatalf("fim on: got %d, want 200; body=%s", resp.Code, resp.Body.String()) + } + if !strings.Contains(resp.Body.String(), `"teams":[]`) { + t.Errorf("body missing \"teams\": %s", resp.Body.String()) + } + if !strings.Contains(resp.Body.String(), `"raids":[]`) { + t.Errorf("body missing \"raids\": %s", resp.Body.String()) + } +} + +func TestHumaStationAvailableRoute(t *testing.T) { + prevSecret := config.Config.ApiSecret + prevFim := config.Config.FortInMemory + config.Config.ApiSecret = "topsecret" + defer func() { config.Config.ApiSecret = prevSecret; config.Config.FortInMemory = prevFim }() + + _, api := humatest.New(t, newHumaConfig("test")) + api.UseMiddleware(golbatSecretMiddleware(api)) + registerFortScanRoutes(api) + + config.Config.FortInMemory = false + if resp := api.Get("/api/station/available", "X-Golbat-Secret: topsecret"); resp.Code != http.StatusServiceUnavailable { + t.Errorf("fim off: got %d, want 503", resp.Code) + } + config.Config.FortInMemory = true + resp := api.Get("/api/station/available", "X-Golbat-Secret: topsecret") + if resp.Code != http.StatusOK { + t.Fatalf("fim on: got %d, want 200; body=%s", resp.Code, resp.Body.String()) + } + if !strings.Contains(resp.Body.String(), `"battles":[]`) { + t.Errorf("body missing empty battles array: %s", resp.Body.String()) + } +} diff --git a/routes_huma.go b/routes_huma.go index 223cbdfd..65d892b8 100644 --- a/routes_huma.go +++ b/routes_huma.go @@ -141,9 +141,18 @@ type pokestopAvailableOutput struct { Body *decoder.ApiAvailablePokestops } +type gymAvailableOutput struct { + Body *decoder.ApiAvailableGyms +} + +type stationAvailableOutput struct { + Body *decoder.ApiAvailableStations +} + // registerFortScanRoutes registers the four in-memory fort scan operations -// plus the pokestop-available aggregate. These are gated by -// config.Config.FortInMemory and return 503 when disabled. +// plus the pokestop-available, gym-available and station-available +// aggregates. These are gated by config.Config.FortInMemory and return 503 +// when disabled. func registerFortScanRoutes(api huma.API) { gymOp := huma.Operation{ OperationID: "scan-gyms", @@ -234,6 +243,42 @@ func registerFortScanRoutes(api huma.API) { } return &pokestopAvailableOutput{Body: decoder.GetAvailablePokestops(time.Now().Unix())}, nil }) + + gymAvailableOp := huma.Operation{ + OperationID: "available-gyms", + Method: http.MethodGet, + Path: "/api/gym/available", + Summary: "List currently available gym teams/slots and raid options", + Description: "Distinct (team, available-slots) pairs and active raid levels/bosses/eggs on resident gyms, from the in-memory fort cache (no DB scan). Whole-instance; requires fort_in_memory (503 otherwise).", + Tags: []string{"Fort"}, + Security: []map[string][]string{{securitySchemeName: {}}}, + DefaultStatus: http.StatusOK, + } + draftBadge(&gymAvailableOp) + huma.Register(api, gymAvailableOp, func(ctx context.Context, _ *struct{}) (*gymAvailableOutput, error) { + if !config.Config.FortInMemory { + return nil, huma.Error503ServiceUnavailable("fort_in_memory not enabled") + } + return &gymAvailableOutput{Body: decoder.GetAvailableGyms(time.Now().Unix())}, nil + }) + + stationAvailableOp := huma.Operation{ + OperationID: "available-stations", + Method: http.MethodGet, + Path: "/api/station/available", + Summary: "List currently available station battle options", + Description: "Distinct active (battle level, pokemon) options on resident stations, from the in-memory fort cache (no DB scan). Whole-instance; requires fort_in_memory (503 otherwise).", + Tags: []string{"Fort"}, + Security: []map[string][]string{{securitySchemeName: {}}}, + DefaultStatus: http.StatusOK, + } + draftBadge(&stationAvailableOp) + huma.Register(api, stationAvailableOp, func(ctx context.Context, _ *struct{}) (*stationAvailableOutput, error) { + if !config.Config.FortInMemory { + return nil, huma.Error503ServiceUnavailable("fort_in_memory not enabled") + } + return &stationAvailableOutput{Body: decoder.GetAvailableStations(time.Now().Unix())}, nil + }) } // maxQueryIDs caps the number of ids accepted by the by-id batch query endpoints. @@ -273,6 +318,11 @@ type gymByIdInput struct { } type gymByIdOutput struct{ Body decoder.ApiGymResult } +type stationByIdInput struct { + StationId string `path:"station_id" doc:"ID of the station"` +} +type stationByIdOutput struct{ Body decoder.ApiStationResult } + type pokestopByIdInput struct { FortId string `path:"fort_id" doc:"Fort ID of the pokestop"` } @@ -506,6 +556,32 @@ func registerTier3Routes(api huma.API) { return &gymByIdOutput{Body: decoder.BuildGymResult(gym)}, nil }) + // GET /api/station/id/{station_id} + huma.Register(api, huma.Operation{ + OperationID: "get-station", + Method: http.MethodGet, + Path: "/api/station/id/{station_id}", + Summary: "Get a single station by id", + Description: "Returns the station with the given id, or 404 if not present.", + Tags: []string{"Fort"}, + Security: []map[string][]string{{securitySchemeName: {}}}, + DefaultStatus: http.StatusAccepted, + }, func(ctx context.Context, in *stationByIdInput) (*stationByIdOutput, error) { + tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + station, unlock, err := decoder.GetStationRecordReadOnly(tctx, dbDetails, in.StationId, "API.GetStation") + if unlock != nil { + defer unlock() + } + cancel() + if err != nil { + return nil, huma.Error500InternalServerError("error retrieving station") + } + if station == nil { + return nil, huma.Error404NotFound("station not found") + } + return &stationByIdOutput{Body: decoder.BuildStationResult(station)}, nil + }) + // GET /api/pokestop/id/{fort_id} huma.Register(api, huma.Operation{ OperationID: "get-pokestop", @@ -518,16 +594,24 @@ func registerTier3Routes(api huma.API) { DefaultStatus: http.StatusAccepted, }, func(ctx context.Context, in *pokestopByIdInput) (*pokestopByIdOutput, error) { pokestop, unlock, err := decoder.PeekPokestopRecord(in.FortId, "API.GetPokestop") - if unlock != nil { - defer unlock() - } if err != nil { + if unlock != nil { + unlock() + } return nil, huma.Error500InternalServerError("error retrieving pokestop") } if pokestop == nil { + if unlock != nil { + unlock() + } return nil, huma.Error404NotFound("pokestop not found") } - return &pokestopByIdOutput{Body: decoder.BuildPokestopResult(pokestop)}, nil + body := decoder.BuildPokestopResult(pokestop) + if unlock != nil { + unlock() // release before locking incidents + } + body.Invasions = decoder.CollectPokestopIncidents(ctx, dbDetails, in.FortId, time.Now().Unix()) + return &pokestopByIdOutput{Body: body}, nil }) // GET /api/tappable/id/{tappable_id} From bfffae7c17f6d09d4ec5196be9c12973add8113d Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 15:38:32 +0100 Subject: [PATCH 02/29] docs: fort map-data and dnf filtering design specs Co-Authored-By: Claude Fable 5 --- .../2026-07-16-fort-scan-map-data-golbat.md | 970 ++++++++++++++++++ .../2026-07-16-fort-dnf-filtering-design.md | 242 +++++ .../2026-07-16-fort-scan-map-data-design.md | 352 +++++++ 3 files changed, 1564 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-16-fort-scan-map-data-golbat.md create mode 100644 docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md create mode 100644 docs/superpowers/specs/2026-07-16-fort-scan-map-data-design.md diff --git a/docs/superpowers/plans/2026-07-16-fort-scan-map-data-golbat.md b/docs/superpowers/plans/2026-07-16-fort-scan-map-data-golbat.md new file mode 100644 index 00000000..dc7953c6 --- /dev/null +++ b/docs/superpowers/plans/2026-07-16-fort-scan-map-data-golbat.md @@ -0,0 +1,970 @@ +# Golbat Fort-Scan Map-Data — Golbat PR Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add the Golbat-side gaps that let ReactMap serve fort map-data (pokestops, gyms, stations) from Golbat: whole-row invasions in the pokestop scan/by-id, a station by-id endpoint, station preload under bare `fort_in_memory`, and gym/station `available` list endpoints. + +**Architecture:** `FortLookup` stays a DNF-only index; responses are whole records from the record caches (`pokestopCache`/`incidentCache`/`gymCache`/`stationCache`). Invasions attach via a plain **string** fetch handle on `FortLookupIncident` (the `int64` re-key is parked → UnownHash/Golbat#384). The two new `available` endpoints are single `fortLookupCache.Range` aggregates mirroring `GetAvailablePokestops`. + +**Tech Stack:** Go 1.26 (module `golbat`), Huma v2 (`humatest` for route tests), `xsync/v4` (`fortLookupCache`), `ottercache` (`incidentCache`), `guregu/null/v6`, logrus. Build tag `go_json`. + +## Global Constraints + +- `FortLookup` carries only what `isFortDnfMatch` reads. The new `FortLookupIncident.Id string` is a **record locator**, not display data — no other display fields added to `FortLookup`. +- Whole records only: invasion payloads are the whole `IncidentData` row from `incidentCache` (read-through to DB on miss via `getIncidentRecordReadOnly`), never a `FortLookup` projection. +- Incident id stays a **string** everywhere (proto `IncidentId` is a string; the `int64` re-key is out of scope → UnownHash/Golbat#384). No DB schema change. +- Scan + `available` routes: `FortInMemory`-gated (`huma.Error503ServiceUnavailable("fort_in_memory not enabled")`), `Security: golbatSecret`, `draftBadge(&op)`, `Tags` per family. **By-id routes are NOT gated** — mirror the existing gym/pokestop by-id (`GetXRecordReadOnly` with DB fallback). (This corrects spec §7.4's "FortInMemory-gated" phrasing, which applies to scan/available, not by-id.) +- `with_incidents` is a body field on the shared `ApiFortScan` (`json:"with_incidents"`, `required:"false"`, default false); only the pokestop + combined scan handlers honor it. +- Lock ordering: the existing `saveIncidentRecord` locks incident → pokestop. Any code that fetches incidents for a pokestop result MUST release the pokestop lock **before** locking incidents, to avoid the reverse order. +- House test style: reset only the package-level cache vars a test touches (`fortLookupCache = xsync.NewMap[string, FortLookup]()`), struct-literal fixtures, plain `if … { t.Fatalf }`, no assertion lib. Decoder tests: `go test ./decoder/... -run TestX -v`. Route tests (package `main`, repo root): `go test . -run TestX -v`. Build: `go build -tags go_json golbat`. + +--- + +### Task 1: String incident fetch handle on `FortLookupIncident` + +**Files:** +- Modify: `decoder/station_battle.go:51-59` (`FortLookupIncident` struct) +- Modify: `decoder/fortRtree.go:267-306` (`updatePokestopIncidentLookup`) +- Test: `decoder/fort_incident_id_test.go` (create) + +**Interfaces:** +- Produces: `FortLookupIncident.Id string` — the incident id, populated on every incident upsert; consumed by Task 2's `CollectPokestopIncidents`. + +- [ ] **Step 1: Write the failing test** — `decoder/fort_incident_id_test.go`: + +```go +package decoder + +import ( + "testing" + + "github.com/guregu/null/v6" + "github.com/puzpuzpuz/xsync/v4" +) + +// updatePokestopIncidentLookup must carry the incident Id onto the FortLookup +// projection so the scan can fetch the whole incident row from incidentCache. +func TestUpdatePokestopIncidentLookupCarriesId(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + const id = "stop-1" + fortLookupCache.Store(id, FortLookup{FortType: POKESTOP, Lat: 1, Lon: 2}) + + inc := &Incident{IncidentData: IncidentData{ + Id: "-1016089077232382347", + DisplayType: 1, + Character: 5, + Confirmed: true, + Slot1PokemonId: null.IntFrom(41), + ExpirationTime: 9_999_999_999, + }} + updatePokestopIncidentLookup(id, inc) + + fl, ok := fortLookupCache.Load(id) + if !ok || len(fl.Incidents) != 1 { + t.Fatalf("expected 1 incident, got %+v", fl.Incidents) + } + if fl.Incidents[0].Id != "-1016089077232382347" { + t.Fatalf("incident Id not carried: %q", fl.Incidents[0].Id) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./decoder/... -run TestUpdatePokestopIncidentLookupCarriesId -v` +Expected: FAIL to compile — `fl.Incidents[0].Id undefined (type FortLookupIncident has no field or method Id)`. + +- [ ] **Step 3: Add the field** — in `decoder/station_battle.go`, add `Id` as the first field of `FortLookupIncident`: + +```go +type FortLookupIncident struct { + Id string // incident id — fetch handle into incidentCache (not DNF-used) + DisplayType int8 + Style int8 + Character int16 + Confirmed bool + Slot1PokemonId int16 + Slot1Form int16 + ExpireTimestamp int64 // used to skip expired incidents at filter time +} +``` + +- [ ] **Step 4: Populate it** — in `decoder/fortRtree.go`, in `updatePokestopIncidentLookup`, add `Id` to the `updated` literal: + +```go + updated := FortLookupIncident{ + Id: incident.Id, + DisplayType: int8(incident.DisplayType), + Style: int8(incident.Style), + Character: incident.Character, + Confirmed: incident.Confirmed, + Slot1PokemonId: int16(incident.Slot1PokemonId.ValueOrZero()), + Slot1Form: int16(incident.Slot1Form.ValueOrZero()), + ExpireTimestamp: incident.ExpirationTime, + } +``` + +- [ ] **Step 5: Run tests to verify they pass** (incl. the existing concurrency + DNF tests, which construct `FortLookupIncident` without `Id` — a leading new field keeps positional literals valid only if they use field names; the existing tests at `fort_incident_test.go` and `api_pokestop_available_test.go` use **named** fields, so they still compile): + +Run: `go test ./decoder/... -run 'TestUpdatePokestopIncidentLookupCarriesId|TestFortDnfMatch_IncidentSlice|TestFortLookupConcurrentPokestopAndIncidentWriters|TestGetAvailablePokestops' -v` +Expected: PASS (4 tests). + +- [ ] **Step 6: Commit** + +```bash +git add decoder/station_battle.go decoder/fortRtree.go decoder/fort_incident_id_test.go +git commit -m "feat(fort): carry incident id on FortLookupIncident as a fetch handle" +``` + +--- + +### Task 2: Invasions in the pokestop scan + by-id (`with_incidents`) + +**Files:** +- Modify: `decoder/api_pokestop.go` (add `ApiPokestopIncident`, `Invasions` field, `buildPokestopIncident`, `CollectPokestopIncidents`) +- Modify: `decoder/api_fort.go:14-19` (`ApiFortScan.WithIncidents`), `:344-367` (`PokestopScanEndpoint`), and the pokestop loop in `FortCombinedScanEndpoint` +- Modify: `routes_huma.go:509-531` (pokestop by-id handler attaches incidents) +- Test: `decoder/api_pokestop_incidents_test.go` (create) + +**Interfaces:** +- Consumes: `FortLookupIncident.Id` (Task 1). +- Produces: `ApiPokestopResult.Invasions []ApiPokestopIncident`; `decoder.CollectPokestopIncidents(ctx context.Context, dbDetails db.DbDetails, fortId string, now int64) []ApiPokestopIncident`; `ApiFortScan.WithIncidents bool`. + +- [ ] **Step 1: Write the failing test** — `decoder/api_pokestop_incidents_test.go`: + +```go +package decoder + +import ( + "context" + "testing" + + db "golbat/db" + + "github.com/guregu/null/v6" + "github.com/puzpuzpuz/xsync/v4" +) + +// CollectPokestopIncidents returns the whole active-incident rows for a fort, +// looked up from incidentCache via the FortLookup handles, skipping expired. +func TestCollectPokestopIncidents(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + incidentCache = newTestIncidentCache() + now := int64(1_000_000) + + active := &Incident{IncidentData: IncidentData{ + Id: "inc-active", PokestopId: "s1", DisplayType: 1, Character: 5, + Confirmed: true, Slot1PokemonId: null.IntFrom(41), ExpirationTime: now + 100, + }} + expired := &Incident{IncidentData: IncidentData{ + Id: "inc-expired", PokestopId: "s1", DisplayType: 3, Character: 30, ExpirationTime: now - 1, + }} + incidentCache.Set("inc-active", active, 0) + incidentCache.Set("inc-expired", expired, 0) + + fortLookupCache.Store("s1", FortLookup{FortType: POKESTOP, Incidents: []FortLookupIncident{ + {Id: "inc-active", DisplayType: 1, Character: 5, ExpireTimestamp: now + 100}, + {Id: "inc-expired", DisplayType: 3, Character: 30, ExpireTimestamp: now - 1}, + }}) + + got := CollectPokestopIncidents(context.Background(), db.DbDetails{}, "s1", now) + if len(got) != 1 { + t.Fatalf("expected 1 active incident, got %d: %+v", len(got), got) + } + if got[0].Id != "inc-active" || got[0].Character != 5 || got[0].Slot1PokemonId == nil || *got[0].Slot1PokemonId != 41 { + t.Fatalf("wrong incident payload: %+v", got[0]) + } +} +``` + +Note: this test needs a helper `newTestIncidentCache()` because `incidentCache` is created in `main.go`'s init path. Add it to the test file: + +```go +func newTestIncidentCache() *ottercache.OtterCache[string, *Incident] { + return ottercache.NewOtterCache(ottercache.OtterCacheConfig[string, *Incident]{ + Name: "incident-test", DefaultTTL: 60 * time.Minute, + }) +} +``` + +with imports `"time"` and `ottercache "golbat/ottercache"` (confirm the import path from `decoder/main.go:203`'s usage; it is imported there as `ottercache`). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./decoder/... -run TestCollectPokestopIncidents -v` +Expected: FAIL to compile — `undefined: CollectPokestopIncidents`, `undefined: ApiPokestopIncident field`. + +- [ ] **Step 3: Add the response type + builders** — append to `decoder/api_pokestop.go`: + +```go +// ApiPokestopIncident is one active incident (whole row) on a pokestop, as +// returned in a scan/by-id response when with_incidents is set. Sourced from +// incidentCache via the FortLookup fetch handle; nullable slots are pointers. +type ApiPokestopIncident struct { + Id string `json:"id" doc:"Incident id"` + DisplayType int16 `json:"display_type" doc:"Incident display type (1-4 rocket, 7 goldstop, 8 kecleon, 9 showcase)"` + Style int16 `json:"style" doc:"Incident style"` + Character int16 `json:"character" doc:"Invasion character id (grunt/leader/giovanni); 0 for non-rocket"` + StartTime int64 `json:"start" doc:"Unix timestamp when the incident started"` + ExpirationTime int64 `json:"expiration" doc:"Unix timestamp when the incident expires"` + Confirmed bool `json:"confirmed" doc:"True when the lineup is confirmed (grunts only)"` + Slot1PokemonId *int64 `json:"slot_1_pokemon_id" doc:"Confirmed lead pokemon id, else null"` + Slot1Form *int64 `json:"slot_1_form" doc:"Confirmed lead pokemon form, else null"` + Slot2PokemonId *int64 `json:"slot_2_pokemon_id" doc:"Slot 2 pokemon id, else null"` + Slot2Form *int64 `json:"slot_2_form" doc:"Slot 2 form, else null"` + Slot3PokemonId *int64 `json:"slot_3_pokemon_id" doc:"Slot 3 pokemon id, else null"` + Slot3Form *int64 `json:"slot_3_form" doc:"Slot 3 form, else null"` +} + +func buildPokestopIncident(inc *Incident) ApiPokestopIncident { + return ApiPokestopIncident{ + Id: inc.Id, + DisplayType: inc.DisplayType, + Style: inc.Style, + Character: inc.Character, + StartTime: inc.StartTime, + ExpirationTime: inc.ExpirationTime, + Confirmed: inc.Confirmed, + Slot1PokemonId: inc.Slot1PokemonId.Ptr(), + Slot1Form: inc.Slot1Form.Ptr(), + Slot2PokemonId: inc.Slot2PokemonId.Ptr(), + Slot2Form: inc.Slot2Form.Ptr(), + Slot3PokemonId: inc.Slot3PokemonId.Ptr(), + Slot3Form: inc.Slot3Form.Ptr(), + } +} + +// CollectPokestopIncidents returns the whole-row active incidents for a fort, +// resolved from incidentCache via the string handles in the fort's FortLookup +// (read-through to DB on the rare cache miss). Callers MUST NOT hold the +// pokestop lock — this locks incidents, and saveIncidentRecord locks +// incident->pokestop, so holding pokestop here would invert the order. +func CollectPokestopIncidents(ctx context.Context, dbDetails db.DbDetails, fortId string, now int64) []ApiPokestopIncident { + fl, ok := fortLookupCache.Load(fortId) + if !ok || len(fl.Incidents) == 0 { + return nil + } + out := make([]ApiPokestopIncident, 0, len(fl.Incidents)) + for _, li := range fl.Incidents { + if li.ExpireTimestamp <= now || li.Id == "" { + continue + } + inc, unlock, err := getIncidentRecordReadOnly(ctx, dbDetails, li.Id, "API.CollectPokestopIncidents") + if err != nil || inc == nil { + if unlock != nil { + unlock() + } + continue + } + out = append(out, buildPokestopIncident(inc)) + unlock() + } + return out +} +``` + +Add the `Invasions` field to `ApiPokestopResult` (after `ShowcaseRankings`): + +```go + ShowcaseRankings *string `json:"showcase_rankings" doc:"Serialized showcase contest rankings"` + Invasions []ApiPokestopIncident `json:"invasions,omitempty" doc:"Active incidents; present only when with_incidents was requested"` +``` + +Ensure `decoder/api_pokestop.go` imports `"context"` and `db "golbat/db"` (match the alias used in `api_fort.go`). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `go test ./decoder/... -run TestCollectPokestopIncidents -v` +Expected: PASS. + +- [ ] **Step 5: Wire `with_incidents` into the request + endpoints.** Add the field to `ApiFortScan` (`decoder/api_fort.go:14-19`): + +```go +type ApiFortScan struct { + Min ApiLatLon `json:"min" doc:"SW (minimum lat/lon) corner of the bounding box."` + Max ApiLatLon `json:"max" doc:"NE (maximum lat/lon) corner of the bounding box."` + Limit int `json:"limit" required:"false" doc:"Max results to return; 0 uses the server default."` + DnfFilters []ApiFortDnfFilter `json:"filters" required:"false" doc:"OR'd filter clauses; a fort matches if it satisfies any one clause. List conditions apply only when present: omit or send null for no constraint — an explicitly empty list matches nothing."` + WithIncidents bool `json:"with_incidents" required:"false" doc:"Pokestop only: when true, each pokestop result includes its active incidents (invasions). Ignored for gym/station."` +} +``` + +In `PokestopScanEndpoint` (`decoder/api_fort.go:344-367`), attach incidents **after releasing the pokestop lock**: + +```go +func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiPokestopScanResult { + returnKeys, examined, skipped, total := internalGetForts(POKESTOP, retrieveParameters) + results := make([]*ApiPokestopResult, 0, len(returnKeys)) + start := time.Now() + now := time.Now().Unix() + + for _, key := range returnKeys { + pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanpokemon") + if err == nil && pokestop != nil { + pokestopCopy := buildPokestopResult(pokestop) + if unlock != nil { + unlock() // release pokestop lock BEFORE locking incidents (lock-order) + unlock = nil + } + if retrieveParameters.WithIncidents { + pokestopCopy.Invasions = CollectPokestopIncidents(context.Background(), dbDetails, key, now) + } + results = append(results, &pokestopCopy) + } + if unlock != nil { + unlock() + } + } + log.Infof("PokestopScan - result buffer time %s, %d added", time.Since(start), len(results)) + + return &ApiPokestopScanResult{ + Pokestops: results, + Examined: examined, + Skipped: skipped, + Total: total, + } +} +``` + +Apply the **same** attach in `FortCombinedScanEndpoint`'s pokestop-building loop (same file): after `buildPokestopResult`, release the pokestop unlock, then `if retrieveParameters.WithIncidents { copy.Invasions = CollectPokestopIncidents(context.Background(), dbDetails, key, now) }`. (Read the current `FortCombinedScanEndpoint` body first; its pokestop loop mirrors `PokestopScanEndpoint`'s.) + +- [ ] **Step 6: Attach incidents in the pokestop by-id handler** — `routes_huma.go:509-531`, restructure to release the pokestop lock before the incident lookup: + +```go + }, func(ctx context.Context, in *pokestopByIdInput) (*pokestopByIdOutput, error) { + pokestop, unlock, err := decoder.PeekPokestopRecord(in.FortId, "API.GetPokestop") + if err != nil { + if unlock != nil { + unlock() + } + return nil, huma.Error500InternalServerError("error retrieving pokestop") + } + if pokestop == nil { + if unlock != nil { + unlock() + } + return nil, huma.Error404NotFound("pokestop not found") + } + body := decoder.BuildPokestopResult(pokestop) + if unlock != nil { + unlock() // release before locking incidents + } + body.Invasions = decoder.CollectPokestopIncidents(ctx, dbDetails, in.FortId, time.Now().Unix()) + return &pokestopByIdOutput{Body: body}, nil + }) +``` + +- [ ] **Step 7: Build + run the full decoder + route suites** + +Run: `go build -tags go_json golbat && go test ./decoder/... -run 'TestCollectPokestopIncidents|TestGetAvailablePokestops|TestFortDnfMatch_IncidentSlice' -v` +Expected: build OK; PASS. + +- [ ] **Step 8: Commit** + +```bash +git add decoder/api_pokestop.go decoder/api_fort.go routes_huma.go decoder/api_pokestop_incidents_test.go +git commit -m "feat(fort): attach whole-row invasions to pokestop scan/by-id via with_incidents" +``` + +--- + +### Task 3: Station by-id endpoint + +**Files:** +- Modify: `routes_huma.go` (add `stationByIdInput`/`stationByIdOutput` near `:276`, register `GET /api/station/id/{station_id}` near the gym by-id at `:483`) +- Test: `huma_routes_test.go` (add a route test) + +**Interfaces:** +- Consumes: `decoder.GetStationRecordReadOnly(ctx, dbDetails, id, caller)` and `decoder.BuildStationResult(station)` (both exist). + +- [ ] **Step 1: Write the failing test** — add to `huma_routes_test.go`: + +```go +// TestHumaStationByIdRoute verifies the new station by-id route is registered, +// requires the secret, and 404s for an unknown id (empty cache, no DB). +func TestHumaStationByIdRoute(t *testing.T) { + prev := config.Config.ApiSecret + config.Config.ApiSecret = "topsecret" + defer func() { config.Config.ApiSecret = prev }() + + _, api := humatest.New(t, newHumaConfig("test")) + api.UseMiddleware(golbatSecretMiddleware(api)) + registerHumaRoutes(api) + + t.Run("no secret is 401", func(t *testing.T) { + resp := api.Get("/api/station/id/does-not-exist") + if resp.Code != http.StatusUnauthorized { + t.Errorf("got %d, want 401", resp.Code) + } + }) + t.Run("unknown id is 404", func(t *testing.T) { + resp := api.Get("/api/station/id/does-not-exist", "X-Golbat-Secret: topsecret") + if resp.Code != http.StatusNotFound { + t.Errorf("got %d, want 404; body=%s", resp.Code, resp.Body.String()) + } + }) +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test . -run TestHumaStationByIdRoute -v` +Expected: FAIL — the unknown-id case returns 404 only if the route exists; before registration it's 404 from the router's default "no operation" or 401 mismatch. Confirm it fails (route missing). + +- [ ] **Step 3: Add the input/output types** — near `routes_huma.go:276`: + +```go +type stationByIdInput struct { + StationId string `path:"station_id" doc:"ID of the station"` +} +type stationByIdOutput struct{ Body decoder.ApiStationResult } +``` + +- [ ] **Step 4: Register the route** — mirror `GET /api/gym/id/{gym_id}` (ungated), placed next to it (~`routes_huma.go:507`): + +```go + // GET /api/station/id/{station_id} + huma.Register(api, huma.Operation{ + OperationID: "get-station", + Method: http.MethodGet, + Path: "/api/station/id/{station_id}", + Summary: "Get a single station by id", + Description: "Returns the station with the given id, or 404 if not present.", + Tags: []string{"Fort"}, + Security: []map[string][]string{{securitySchemeName: {}}}, + DefaultStatus: http.StatusAccepted, + }, func(ctx context.Context, in *stationByIdInput) (*stationByIdOutput, error) { + tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + station, unlock, err := decoder.GetStationRecordReadOnly(tctx, dbDetails, in.StationId, "API.GetStation") + if unlock != nil { + defer unlock() + } + cancel() + if err != nil { + return nil, huma.Error500InternalServerError("error retrieving station") + } + if station == nil { + return nil, huma.Error404NotFound("station not found") + } + return &stationByIdOutput{Body: decoder.BuildStationResult(station)}, nil + }) +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `go build -tags go_json golbat && go test . -run TestHumaStationByIdRoute -v` +Expected: build OK; PASS (401 then 404). + +- [ ] **Step 6: Commit** + +```bash +git add routes_huma.go huma_routes_test.go +git commit -m "feat(fort): add GET /api/station/id/{id} single-station endpoint" +``` + +--- + +### Task 4: Station preload under bare `fort_in_memory` + +**Files:** +- Modify: `decoder/preload.go:60-84` (`PreloadForts`) + +**Interfaces:** +- Consumes: `preloadStations(dbDetails, populateRtree)` and `preloadStationBattles(dbDetails, populateRtree)` (both exist). + +Note: this loads from the DB, so it is verified by build + a documented integration smoke, not a unit test (the existing `preload*` functions have no unit tests — they require a live DB). The ordering constraint is real: `preloadStationBattles` needs `stationCache` populated first (`station_battle.go:686` checks `stationCache.Has`). + +- [ ] **Step 1: Add station loading to `PreloadForts`**, preserving the two-phase order (stations before battles), mirroring `Preload`: + +```go +func PreloadForts(dbDetails db.DbDetails, populateRtree bool) error { + startTime := time.Now() + + var wg sync.WaitGroup + var pokestopCount, gymCount, stationCount int32 + + // Phase 1: forts (pokestops, gyms, stations) in parallel. + wg.Add(3) + go func() { + defer wg.Done() + pokestopCount = preloadPokestops(dbDetails, populateRtree) + }() + go func() { + defer wg.Done() + gymCount = preloadGyms(dbDetails, populateRtree) + }() + go func() { + defer wg.Done() + stationCount = preloadStations(dbDetails, populateRtree) + }() + wg.Wait() + + // Phase 2: station battles depend on stationCache being populated. + stationBattleCount := preloadStationBattles(dbDetails, populateRtree) + + log.Infof("PreloadForts: loaded %d pokestops, %d gyms, %d stations, %d station battles in %v (rtree=%v)", + pokestopCount, gymCount, stationCount, stationBattleCount, time.Since(startTime), populateRtree) + + return nil +} +``` + +- [ ] **Step 2: Build** + +Run: `go build -tags go_json golbat` +Expected: OK. + +- [ ] **Step 3: Documented integration smoke** (record in the PR, not a unit test): start Golbat with `fort_in_memory = true` and `preload = false`; confirm the log line `PreloadForts: loaded … stations, … station battles`; `curl -H "X-Golbat-Secret: " -XPOST /api/station/scan -d '{"min":…,"max":…}'` returns stations in a scanned area (empty before this change). + +- [ ] **Step 4: Commit** + +```bash +git add decoder/preload.go +git commit -m "feat(preload): load stations + battles under bare fort_in_memory" +``` + +--- + +### Task 5: Gym `available` endpoint + +**Files:** +- Create: `decoder/api_gym_available.go` +- Modify: `routes_huma.go` (register `GET /api/gym/available` in `registerFortScanRoutes`, add `gymAvailableOutput`) +- Test: `decoder/api_gym_available_test.go` (create) + +**Interfaces:** +- Produces: `decoder.GetAvailableGyms(now int64) *ApiAvailableGyms`. + +- [ ] **Step 1: Write the failing test** — `decoder/api_gym_available_test.go`: + +```go +package decoder + +import ( + "testing" + + "github.com/puzpuzpuz/xsync/v4" +) + +func TestGetAvailableGyms(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + now := int64(1_000_000) + + // gym with team + active raid boss + fortLookupCache.Store("g1", FortLookup{ + FortType: GYM, TeamId: 1, AvailableSlots: 2, + RaidLevel: 5, RaidPokemonId: 150, RaidPokemonForm: 0, RaidEndTimestamp: now + 100, + }) + // gym with an active egg (no boss) and an EXPIRED raid on another + fortLookupCache.Store("g2", FortLookup{ + FortType: GYM, TeamId: 2, AvailableSlots: 6, + RaidLevel: 3, RaidPokemonId: 0, RaidEndTimestamp: now + 100, + }) + fortLookupCache.Store("g3", FortLookup{ + FortType: GYM, TeamId: 1, AvailableSlots: 0, + RaidLevel: 5, RaidPokemonId: 999, RaidEndTimestamp: now - 1, // expired -> excluded + }) + // a pokestop must be ignored + fortLookupCache.Store("s1", FortLookup{FortType: POKESTOP, LureId: 501}) + + res := GetAvailableGyms(now) + + if len(res.Teams) != 3 { // (1,2),(2,6),(1,0) + t.Fatalf("teams: %+v", res.Teams) + } + // raids: boss 150 lvl5, egg lvl3; expired 999 excluded + var bosses, eggs int + for _, r := range res.Raids { + if r.PokemonId == 999 { + t.Fatalf("expired raid leaked: %+v", r) + } + if r.PokemonId == 0 { + eggs++ + } else { + bosses++ + } + } + if bosses != 1 || eggs != 1 { + t.Fatalf("raids: %+v", res.Raids) + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `go test ./decoder/... -run TestGetAvailableGyms -v` +Expected: FAIL — `undefined: GetAvailableGyms`. + +- [ ] **Step 3: Implement** — `decoder/api_gym_available.go`: + +```go +package decoder + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +// ApiGymTeamAvailable is one distinct (team, available-slots) pair present on +// resident gyms, with how many gyms carry it. ReactMap derives its t/g keys. +type ApiGymTeamAvailable struct { + TeamId int8 `json:"team_id" doc:"Controlling team id (0 = uncontested)"` + AvailableSlots int8 `json:"available_slots" doc:"Open defender slots"` + Count int `json:"count" doc:"Number of resident gyms with this team/slots"` +} + +// ApiGymRaidAvailable is one distinct active raid option on resident gyms. +// PokemonId 0 means an egg (no boss yet). ReactMap derives its e/r/boss keys. +type ApiGymRaidAvailable struct { + RaidLevel int8 `json:"raid_level" doc:"Raid level/tier"` + PokemonId int16 `json:"pokemon_id" doc:"Raid boss pokemon id; 0 = egg (unhatched)"` + Form int16 `json:"form" doc:"Raid boss form id, else 0"` + Count int `json:"count" doc:"Number of resident gyms with this raid option"` +} + +// ApiAvailableGyms is the whole-instance gym filter snapshot served by +// GET /api/gym/available. +type ApiAvailableGyms struct { + Teams []ApiGymTeamAvailable `json:"teams" doc:"Distinct team + available-slot pairs on resident gyms"` + Raids []ApiGymRaidAvailable `json:"raids" doc:"Distinct active raid levels/bosses/eggs on resident gyms"` +} + +// GetAvailableGyms builds the gym filter snapshot from a single fortLookupCache +// range over resident gyms — no maintained map (FortLookup carries every gym +// filter field). Teams are all-resident (no time filter); raids require an +// unexpired raid with level > 0. +func GetAvailableGyms(now int64) *ApiAvailableGyms { + start := time.Now() + res := &ApiAvailableGyms{Teams: []ApiGymTeamAvailable{}, Raids: []ApiGymRaidAvailable{}} + teams := map[ApiGymTeamAvailable]int{} + raids := map[ApiGymRaidAvailable]int{} + forts := 0 + + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + if fl.FortType != GYM { + return true + } + forts++ + teams[ApiGymTeamAvailable{TeamId: fl.TeamId, AvailableSlots: fl.AvailableSlots}]++ + if fl.RaidLevel > 0 && fl.RaidEndTimestamp > now { + raids[ApiGymRaidAvailable{RaidLevel: fl.RaidLevel, PokemonId: fl.RaidPokemonId, Form: fl.RaidPokemonForm}]++ + } + return true + }) + + for k, n := range teams { + k.Count = n + res.Teams = append(res.Teams, k) + } + for k, n := range raids { + k.Count = n + res.Raids = append(res.Raids, k) + } + + if statsCollector != nil { + statsCollector.ObserveApiScan("available-gyms", time.Since(start).Seconds()) + } + log.Infof("available-gyms built in %s: scanned %d gyms -> %d team/slot, %d raid options", + time.Since(start), forts, len(res.Teams), len(res.Raids)) + return res +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `go test ./decoder/... -run TestGetAvailableGyms -v` +Expected: PASS. + +- [ ] **Step 5: Register the route** — in `routes_huma.go`, add near the pokestop-available registration (`:220-236`), inside `registerFortScanRoutes`: + +```go +type gymAvailableOutput struct { + Body *decoder.ApiAvailableGyms +} +``` +(place with the other `*Output` type decls, ~`:140`), then the handler: +```go + gymAvailableOp := huma.Operation{ + OperationID: "available-gyms", + Method: http.MethodGet, + Path: "/api/gym/available", + Summary: "List currently available gym teams/slots and raid options", + Description: "Distinct (team, available-slots) pairs and active raid levels/bosses/eggs on resident gyms, from the in-memory fort cache (no DB scan). Whole-instance; requires fort_in_memory (503 otherwise).", + Tags: []string{"Fort"}, + Security: []map[string][]string{{securitySchemeName: {}}}, + DefaultStatus: http.StatusOK, + } + draftBadge(&gymAvailableOp) + huma.Register(api, gymAvailableOp, func(ctx context.Context, _ *struct{}) (*gymAvailableOutput, error) { + if !config.Config.FortInMemory { + return nil, huma.Error503ServiceUnavailable("fort_in_memory not enabled") + } + return &gymAvailableOutput{Body: decoder.GetAvailableGyms(time.Now().Unix())}, nil + }) +``` + +- [ ] **Step 6: Add a route gating test** — append to `huma_routes_test.go`: + +```go +func TestHumaGymAvailableRoute(t *testing.T) { + prevSecret := config.Config.ApiSecret + prevFim := config.Config.FortInMemory + config.Config.ApiSecret = "topsecret" + defer func() { config.Config.ApiSecret = prevSecret; config.Config.FortInMemory = prevFim }() + + _, api := humatest.New(t, newHumaConfig("test")) + api.UseMiddleware(golbatSecretMiddleware(api)) + registerHumaRoutes(api) + + config.Config.FortInMemory = false + if resp := api.Get("/api/gym/available", "X-Golbat-Secret: topsecret"); resp.Code != http.StatusServiceUnavailable { + t.Errorf("fim off: got %d, want 503", resp.Code) + } + config.Config.FortInMemory = true + resp := api.Get("/api/gym/available", "X-Golbat-Secret: topsecret") + if resp.Code != http.StatusOK { + t.Fatalf("fim on: got %d, want 200; body=%s", resp.Code, resp.Body.String()) + } + for _, key := range []string{"teams", "raids"} { + if !strings.Contains(resp.Body.String(), `"`+key+`"`) { + t.Errorf("body missing %q: %s", key, resp.Body.String()) + } + } +} +``` + +- [ ] **Step 7: Build + test** + +Run: `go build -tags go_json golbat && go test ./decoder/... -run TestGetAvailableGyms -v && go test . -run TestHumaGymAvailableRoute -v` +Expected: build OK; both PASS. + +- [ ] **Step 8: Commit** + +```bash +git add decoder/api_gym_available.go decoder/api_gym_available_test.go routes_huma.go huma_routes_test.go +git commit -m "feat(fort): add GET /api/gym/available (team/slots + raid aggregate)" +``` + +--- + +### Task 6: Station `available` endpoint + +**Files:** +- Create: `decoder/api_station_available.go` +- Modify: `routes_huma.go` (register `GET /api/station/available`, add `stationAvailableOutput`) +- Test: `decoder/api_station_available_test.go` (create) + +**Interfaces:** +- Produces: `decoder.GetAvailableStations(now int64) *ApiAvailableStations`. + +The aggregate mirrors `isFortDnfMatch`'s station branch (`api_fort.go:215-240`): iterate `StationBattles` when non-empty, else fall back to the top-battle projection; skip expired (`BattleEndTimestamp <= now`) and level-0 battles (ReactMap excludes `!battle_level`). + +- [ ] **Step 1: Write the failing test** — `decoder/api_station_available_test.go`: + +```go +package decoder + +import ( + "testing" + + "github.com/puzpuzpuz/xsync/v4" +) + +func TestGetAvailableStations(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + now := int64(1_000_000) + + // station with two active battles (multi-battle path) + one expired + fortLookupCache.Store("st1", FortLookup{FortType: STATION, StationBattles: []FortLookupStationBattle{ + {BattleLevel: 3, BattlePokemonId: 150, BattlePokemonForm: 0, BattleEndTimestamp: now + 100}, + {BattleLevel: 5, BattlePokemonId: 384, BattlePokemonForm: 0, BattleEndTimestamp: now + 100}, + {BattleLevel: 1, BattlePokemonId: 1, BattleEndTimestamp: now - 1}, // expired -> excluded + }}) + // station with only the top-battle projection (no StationBattles slice) + fortLookupCache.Store("st2", FortLookup{FortType: STATION, + BattleLevel: 6, BattlePokemonId: 999, BattlePokemonForm: 0, BattleEndTimestamp: now + 100, + }) + // station with a level-0 battle -> excluded + fortLookupCache.Store("st3", FortLookup{FortType: STATION, StationBattles: []FortLookupStationBattle{ + {BattleLevel: 0, BattlePokemonId: 5, BattleEndTimestamp: now + 100}, + }}) + fortLookupCache.Store("g1", FortLookup{FortType: GYM, TeamId: 1}) // ignored + + res := GetAvailableStations(now) + // expect: (3,150),(5,384) from st1, (6,999) from st2 = 3 distinct; expired + level-0 excluded + if len(res.Battles) != 3 { + t.Fatalf("battles: %+v", res.Battles) + } + for _, b := range res.Battles { + if b.BattleLevel == 0 || b.PokemonId == 1 { + t.Fatalf("excluded battle leaked: %+v", b) + } + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `go test ./decoder/... -run TestGetAvailableStations -v` +Expected: FAIL — `undefined: GetAvailableStations`. + +- [ ] **Step 3: Implement** — `decoder/api_station_available.go`: + +```go +package decoder + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +// ApiStationBattleAvailable is one distinct active (battle_level, pokemon, form) +// option on resident stations. ReactMap derives its - and j keys. +type ApiStationBattleAvailable struct { + BattleLevel int8 `json:"battle_level" doc:"Max battle level"` + PokemonId int16 `json:"pokemon_id" doc:"Battle pokemon id, else 0"` + Form int16 `json:"form" doc:"Battle pokemon form id, else 0"` + Count int `json:"count" doc:"Number of resident stations with this active battle option"` +} + +// ApiAvailableStations is the whole-instance station filter snapshot served by +// GET /api/station/available. +type ApiAvailableStations struct { + Battles []ApiStationBattleAvailable `json:"battles" doc:"Distinct active battle level/pokemon options on resident stations"` +} + +// GetAvailableStations builds the station filter snapshot from a single +// fortLookupCache range. Mirrors isFortDnfMatch's station branch: iterate the +// StationBattles slice when present, else fall back to the top-battle +// projection; skip expired and level-0 battles. +func GetAvailableStations(now int64) *ApiAvailableStations { + start := time.Now() + res := &ApiAvailableStations{Battles: []ApiStationBattleAvailable{}} + battles := map[ApiStationBattleAvailable]int{} + forts := 0 + + add := func(level int8, pokemonId, form int16, end int64) { + if level == 0 || end <= now { + return + } + battles[ApiStationBattleAvailable{BattleLevel: level, PokemonId: pokemonId, Form: form}]++ + } + + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + if fl.FortType != STATION { + return true + } + forts++ + if len(fl.StationBattles) == 0 { + add(fl.BattleLevel, fl.BattlePokemonId, fl.BattlePokemonForm, fl.BattleEndTimestamp) + return true + } + for _, b := range fl.StationBattles { + add(b.BattleLevel, b.BattlePokemonId, b.BattlePokemonForm, b.BattleEndTimestamp) + } + return true + }) + + for k, n := range battles { + k.Count = n + res.Battles = append(res.Battles, k) + } + + if statsCollector != nil { + statsCollector.ObserveApiScan("available-stations", time.Since(start).Seconds()) + } + log.Infof("available-stations built in %s: scanned %d stations -> %d battle options", + time.Since(start), forts, len(res.Battles)) + return res +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `go test ./decoder/... -run TestGetAvailableStations -v` +Expected: PASS. + +- [ ] **Step 5: Register the route** — in `routes_huma.go`, `registerFortScanRoutes`, mirroring Task 5: + +```go +type stationAvailableOutput struct { + Body *decoder.ApiAvailableStations +} +``` +then: +```go + stationAvailableOp := huma.Operation{ + OperationID: "available-stations", + Method: http.MethodGet, + Path: "/api/station/available", + Summary: "List currently available station battle options", + Description: "Distinct active (battle level, pokemon) options on resident stations, from the in-memory fort cache (no DB scan). Whole-instance; requires fort_in_memory (503 otherwise).", + Tags: []string{"Fort"}, + Security: []map[string][]string{{securitySchemeName: {}}}, + DefaultStatus: http.StatusOK, + } + draftBadge(&stationAvailableOp) + huma.Register(api, stationAvailableOp, func(ctx context.Context, _ *struct{}) (*stationAvailableOutput, error) { + if !config.Config.FortInMemory { + return nil, huma.Error503ServiceUnavailable("fort_in_memory not enabled") + } + return &stationAvailableOutput{Body: decoder.GetAvailableStations(time.Now().Unix())}, nil + }) +``` + +- [ ] **Step 6: Add a route gating test** — append to `huma_routes_test.go` (mirror `TestHumaGymAvailableRoute`, asserting the `"battles"` key and 503-when-off): + +```go +func TestHumaStationAvailableRoute(t *testing.T) { + prevSecret := config.Config.ApiSecret + prevFim := config.Config.FortInMemory + config.Config.ApiSecret = "topsecret" + defer func() { config.Config.ApiSecret = prevSecret; config.Config.FortInMemory = prevFim }() + + _, api := humatest.New(t, newHumaConfig("test")) + api.UseMiddleware(golbatSecretMiddleware(api)) + registerHumaRoutes(api) + + config.Config.FortInMemory = false + if resp := api.Get("/api/station/available", "X-Golbat-Secret: topsecret"); resp.Code != http.StatusServiceUnavailable { + t.Errorf("fim off: got %d, want 503", resp.Code) + } + config.Config.FortInMemory = true + resp := api.Get("/api/station/available", "X-Golbat-Secret: topsecret") + if resp.Code != http.StatusOK { + t.Fatalf("fim on: got %d, want 200; body=%s", resp.Code, resp.Body.String()) + } + if !strings.Contains(resp.Body.String(), `"battles"`) { + t.Errorf("body missing battles: %s", resp.Body.String()) + } +} +``` + +- [ ] **Step 7: Build + full suite** + +Run: `go build -tags go_json golbat && go test ./decoder/... && go test . -run TestHuma -v` +Expected: build OK; all PASS. + +- [ ] **Step 8: Commit** + +```bash +git add decoder/api_station_available.go decoder/api_station_available_test.go routes_huma.go huma_routes_test.go +git commit -m "feat(fort): add GET /api/station/available (battle option aggregate)" +``` + +--- + +## Self-Review + +**Spec coverage** (§7.1–7.6 of `2026-07-16-fort-scan-map-data-design.md`): +- §7.1 string incident handle → Task 1. §7.2 invasions in scan/by-id + `with_incidents` → Task 2. §7.3 (no FortLookup widening) → honored (only a string locator added). §7.4 station by-id → Task 3. §7.5 station preload → Task 4. §7.6 gym/station available → Tasks 5–6. Instrumentation (`ObserveApiScan`) → in Tasks 5–6; the scan endpoints were already only `log.Infof` (unchanged). No gap. + +**Placeholder scan:** every code step has complete code; the two DB-dependent verifications (Task 4 preload, and the live golden checks) are explicitly documented as integration smokes, not faked unit tests. + +**Type consistency:** `FortLookupIncident.Id` (Task 1) is consumed by `CollectPokestopIncidents` (Task 2); `ApiPokestopResult.Invasions []ApiPokestopIncident` matches the builder; `ApiFortScan.WithIncidents` is read in `PokestopScanEndpoint` + by-id; `GetAvailableGyms`/`GetAvailableStations` return `*ApiAvailableGyms`/`*ApiAvailableStations` matching the route `*Output` bodies. Route tests use the confirmed harness (`humatest.New` + `registerHumaRoutes` + `golbatSecretMiddleware`). + +**Cross-task ordering:** Task 2 depends on Task 1 (`Id` field). Tasks 3–6 are independent of 1–2 and of each other. Suggested order 1→2→3→4→5→6 groups the incident work first, then the additive routes. + +**Open verification for the implementer:** in Task 2 Step 5, read the current `FortCombinedScanEndpoint` body before editing (its pokestop loop was not quoted here); apply the identical lock-release-then-`CollectPokestopIncidents` change. In Task 2 Step 3, confirm `decoder/api_pokestop.go`'s import block gains `context` + the `db` alias used elsewhere in the package. diff --git a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md new file mode 100644 index 00000000..53fedea7 --- /dev/null +++ b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md @@ -0,0 +1,242 @@ +# Fort DNF Filtering — Design Spec + +- **Date:** 2026-07-16 +- **Status:** Approved design → planning +- **Golbat branch:** `feat/fort-scan-map-data` (worktree `~/GolandProjects/Golbat-wt/pokestop-available-api`), PR #385 — still open +- **ReactMap branch:** `feat/fort-consumer`, PR #1228 — still open +- **Author:** James Berry (with Claude) +- **Extends:** `2026-07-16-fort-scan-map-data-design.md` — this is its "Phase 2 (DNF)", the payoff phase after match-all shipped. + +## 1. Problem + +`Pokestop/Gym/Station.getAll` now fetch map markers from Golbat's fort-scan endpoints, but send +`filters: []` (match-all): Golbat returns **every fort in the viewport** and ReactMap's `secondaryFilter` +does 100% of the narrowing in JS. In dense cities a specific-item filter (a rare quest reward, a raid +boss, a battle pokemon) still ships the whole viewport over the wire. DNF pushes that narrowing into +Golbat's rtree scan so it returns a fraction of the forts. + +## 2. What already exists (Golbat, PR #385) + +`ApiFortDnfFilter` + `isFortDnfMatch` + the `FortLookup` index are implemented and wired into all four +scan endpoints. Today they already DNF-match: + +- **Pokestop:** `lure_id`, `quest_reward_type`, `quest_reward_item_id`, `quest_reward_pokemon` (id+form), + `quest_reward_amount` (min/max) — matched against **either** the AR or non-AR quest; `incident_character`, + `incident_display_type`, `incident_style`, `incident_pokemon` (slot-1 id+form); `contest_pokemon` (id+form), + `contest_pokemon_type`, `contest_total_entries`. +- **Gym:** `team_id`, `available_slots` (min/max), `raid_level`, `raid_pokemon` (id+form) — raid fields only + match gyms with an active raid. +- **Station:** `battle_level`, `battle_pokemon` (id+form) — matches the multi-battle list; only stations + with an active battle. +- **Shared:** `power_up_level` (min/max), `is_ar_scan_eligible` (true only; `false` is a no-op — irrelevant, + ReactMap only ever filters *for* AR-eligible). + +**Request body** (unchanged — the mem branches already post this shape): `{ min, max, limit, with_incidents, +filters: [ApiFortDnfFilter] }`. `filters` is **OR across clauses, AND within a clause**; a null/omitted +list field inside a clause = no constraint; **an empty/omitted top-level `filters` array = match all forts +of that type** (`api_fort.go:288-290`). Response envelope carries `examined` (forts examined in the +viewport), `skipped` (cache misses), `total` (whole-index size), and the matched `[]` array. + +## 3. Architecture — DNF narrows, `secondaryFilter` finalizes + +**The safety model:** `secondaryFilter` (and the station JS gate) **stays and runs after every fetch**. +DNF is therefore a best-effort **superset** narrow; `secondaryFilter` guarantees exactness. Two +consequences: + +1. **Correctness is never at risk from an imperfect DNF translation.** Anything DNF over-returns, + `secondaryFilter` drops. The single hard invariant: a DNF translation must **never be stricter** than + the real filter (never under-return / drop a fort that should show). +2. **Anything DNF can't express stays in `secondaryFilter` as residual** — cleanly, no new code. That is + where quest **title/target** (`adv` substring set-membership), raid/battle **gender**, gym + **ex-eligible/in-battle**, invasion **confirmed**, and station **upcoming/time-window** gates live. + +### 3.1 The poisoning rule (load-bearing) + +ReactMap's fort filters combine with **OR** (a fort shows if it matches *any* active category). So a +backend may emit narrowing clauses **only if it can express every active category**. If any active +category is a match-all toggle (`onlyAllPokestops`, `onlyGyms`, `onlyAllStations`, …) or an +unexpressible gap (gym ex/in-battle, invasion-confirmed), the backend returns **`[]` (match-all)** for +the whole query — a correct superset; `secondaryFilter` narrows. Otherwise it emits **one clause per +active category** (OR-across). This keeps the superset invariant airtight. + +## 4. ReactMap — three fort filter backends + +New pure modules under `server/src/filters/fort/`: `pokestop.js`, `gym.js`, `station.js`, each exporting +`buildDnfFilters(filters, ctx) → ApiFortDnfFilter[]`. Pure/dependency-light (mirrors `PkmnBackend`'s +`buildApiFilter` shape but without the PVP/IV class machinery), node-golden testable like the mappers. +Each `getAll` mem branch swaps `filters: []` for `buildDnfFilters(args.filters, ctx)`. +`secondaryFilter` is **untouched**. + +**Clause shape** (matches `ApiFortDnfFilter` json tags): a JS object per active category, with only the +constrained fields set (unset = unconstrained); id+form pairs as `{ pokemon_id, form }` (form omitted = +any form); ranges as `{ min, max }`. + +### 4.1 Per-type translation (into existing Golbat fields) + +**Pokestop** (`Pokestop.js` `secondaryFilter` key switch is the source of truth): +- `l` → `lure_id: [id]` +- quest reward keys → one clause with `quest_reward_type`/`quest_reward_item_id`/`quest_reward_pokemon`: + `q`→type 2 + `quest_reward_item_id:[item]`; `d`→type 3; `p`→type 1; `c`→type 4 + + `quest_reward_pokemon:[{pk}]`; `x`→type 9 + pokemon; `m-`→type 12 + pokemon; bare + `[-]`→type 7 + `quest_reward_pokemon:[{pk,form}]`; `u`→`quest_reward_type:[type]`. + **`adv` title/target is dropped from the clause** (residual). When `.all` is false and `adv` is present + the clause is a strict superset; `secondaryFilter` applies the title/target substring check. +- `i`→`incident_character:[char]`; `b`→`incident_display_type:[type]`; + `a-`→`incident_pokemon:[{pk,form}]` (residual `confirmed` check stays JS). +- `f-`→`contest_pokemon:[{pk,form}]`; `h`→`contest_pokemon_type:[type]`. +- `onlyArEligible` → `is_ar_scan_eligible: true`; `onlyLevels` (power-up) → `power_up_level:{min,max}`. +- `onlyAllPokestops` → match-all (`[]`). + +**Gym** (`Gym.js` `secondaryFilter` key switch): +- `t-0` → `team_id:[team]`; `g-` → `team_id:[team]` + `available_slots:{min,max}` from the + slot index; `e` → `raid_level:[tiers…]`; bare `-` → `raid_pokemon:[{id,form}]` (gender → + residual). Ignore the dead `r` keys (unused in `getAll`). +- `onlyArEligible`→`is_ar_scan_eligible:true`; `onlyLevels`→`power_up_level`. +- `onlyGyms`/`onlyAllGyms`, `onlyExEligible`, `onlyInBattle` → match-all (`[]`) — the last two are gaps + (residual). + +**Station** (`Station.js` `matchesStationBattleFilter`/key parsing): +- `onlyBattleTier`/`j` → `battle_level:[lvls…]`; bare `-` → `battle_pokemon:[{id,form}]` + (gender → residual). +- `onlyGmaxStationed` → **new** `stationed_gmax: true` (§5) — a direct `total_stationed_gmax > 0` column + test, DNF-clean. +- `onlyInactiveStations` / the active-vs-inactive gate, `onlyIncludeUpcoming`, and the + `battle_start<=ts`/`battle_end>ts` windows → **residual**. These are **now-relative time-window** + predicates (`Station.js:745-954`: `end_time`/`updated` vs `activeCutoff`/`inactiveCutoff`), which DNF's + static filter fields cannot express; `secondaryFilter`'s `passesTimeGate` keeps applying them. (The + `is_inactive` *column* is not the active/inactive gate `getAll` uses, so it is deliberately **not** a + DNF field.) +- `onlyMaxBattles` alone with no per-battle key → `[{station_active:true}]`; `onlyAllStations` → + `[{station_active:true}]` (All-Stations mode only ever shows ACTIVE stations — no poison needed). +- **`station_active` (added post-live-testing):** stations are the one ephemeral fort type; expired + stations accumulate in the index, and a match-all scan shipped 1330 stations of which 174 were live + (−1156 residual). `station_active:true` (Golbat: `StationEndTimestamp > now`, mirroring the + raid/lure/battle now-gating) is stamped into every station clause. The `updated > activeCutoff` + config cutoff and the inactive mode's day-based cutoff stay residual; `onlyInactiveStations` still + poisons (it needs expired stations OR filtered actives). + +## 5. Golbat station gap-fill (folds into #385) + +**One** new station DNF dimension, following the established pattern (`FortLookup` field + populator in +`updateStationLookup`/`updateStationLookupWithBattles` + `ApiFortDnfFilter` field + `isFortDnfMatch` +clause + golden snapshot): + +- **`stationed_gmax *bool`** (`ApiFortDnfFilter`) → when `true`, matches stations with + `FortLookup.TotalStationedGmax > 0` (new `int16` field, populated from `Station.TotalStationedGmax`). + This is a direct, now-independent column test that matches `getAll`'s `onlyGmaxStationed` gate exactly. + +`is_inactive` is **not** filled: `getAll`'s active/inactive gate is a now-relative time-window +computation (§4.1), not the `is_inactive` column, so a column filter would under-return. It stays +residual. Gym ex/in-battle, raid/battle gender, and invasion-confirmed also remain residual (§8). + +**Live-testing postmortem — exact key semantics (the `-997` residual).** The observed 1030→10 quest +residual was NOT staleness (verified: zero stale/expired/NULL-expiry quests in the DB; the clearing +routines refresh DB, record cache and FortLookup together). A `quest_seen_after` freshness gate was +briefly added on that wrong theory and **reverted** (`503a4c5`). The true cause was in the ReactMap +translation: ReactMap quest keys are **exact** — a bare `` key means "reward carries no form_id" +(`secondaryFilter` computes a bare key only when `quest_form_id` is null), and users accumulate +thousands of enabled keys from past rotations (client `deepMerge` never prunes). The backend +translated bare keys to `{pokemon_id}` — Golbat's **any-form wildcard** — so stale keys matched every +form of the species, including the current rotation's (`25-2825`, …): 997 over-returned stops that +exact-key matching then dropped. Fix (`50653c77`): **form-exact pairs everywhere** — bare/formless → +`form:0` (the pokemon-API pattern: proto `FORM_UNSET`=0, `FortLookup` NULL→0), explicit `-` +→ exact — and **one clause per reward type** (2/4/7/9/12 + type-only) so e.g. a candy pair can't +cross-match an encounter stop. No Golbat change needed; no +availability coupling (an enabled∩available intersection was considered and rejected — it would +inherit the availability refresh window as an under-return risk). + +Two further exactness gaps surfaced and were closed in the same investigation: + +- **Form-pinning on form-agnostic keys (latent under-return).** Candy/xl/mega keys (`c25`/`x64`/ + `m150-150`) carry **no form component**, so their match is form-agnostic by construction — the exact + translation is the form **wildcard** (form omitted), not `form:0`. Pinning `form:0` would fail to + match a formed reward if one ever appeared, while the key would match it. The fort matcher's + convention was verified identical to the pokemon API's (`Form *int16`, null = any, set = exact; the + `-1` in the pokemon v2/v3 scanners is an internal bucket sentinel, never a wire value). +- **Historic `-0` encounter keys (the last −56).** Older ReactMap generated `id-0` quest keys; + current code normalizes form-0 to a bare key. `id-0` only matches a stop whose reward carries an + EXPLICIT `form_id: 0`, which reward JSON never contains (verified: 0 rows) — the keys are dead. But + translated as `{id, form:0}` they collide with Golbat's NULL→0 form collapse and match every + formless stop of the species. They are **dropped** from clauses (same accepted-divergence class as + the availableMapper §form note). Amounts are also exact where the key carries one: mega keys group + into per-amount clauses with `quest_reward_amount {amt,amt}`; `d`/`p` emit one + amount-exact clause each; `u` stays type-level by design. + +**Verified live (2026-07-17):** `DNF(5): 8 matched → 8 after secondaryFilter (−0 residual)`, drop +reasons all zero — exact key parity end-to-end (previously 1030 matched → 10, −1020 residual). From +2373 forts scanned, Golbat ships only the rendered set. + +## 6. Observability — the DNF-tuning log + +Each fort mem branch, DNF path, after `secondaryFilter`, logs the two filter stages so the DNF gap is +visible per query: + +``` +[POKESTOP] DNF( clauses): in viewport, − by DNF → , − by secondaryFilter → final +``` + +- ` clauses` (0 = match-all sent) from the backend output length. +- `examined` from the response envelope; `returned` = `res..length`; `final` = post-`secondaryFilter`. +- A large **`−… by secondaryFilter`** (residual drop) flags a filter combination where DNF is leaving + narrowing on the table — the signal for whether to close a gap for that case. A match-all query + (`0 clauses`) with a big residual drop is the clearest "should this become a DNF field?" candidate. + +Emit at `log.info` on the DNF path (replacing/extending the existing per-type info line); keep the SQL +fallback warnings as they are. + +## 7. Testing + +- **Backend unit goldens** (pure, plain `node`): per type, assert (a) each filter-key family produces the + expected clause; (b) the **poisoning rule** — a match-all toggle or a gap category present ⇒ `[]`; (c) + the **superset invariant** on the tricky cases (quest `adv` present ⇒ clause omits title/target; + gender present ⇒ clause omits gender). No test framework; throwaway goldens + eslint/prettier. +- **Golbat**: a small `isFortDnfMatch` unit test for the new `stationed_gmax` field (matches when + `TotalStationedGmax > 0`, wildcard when null). No new `Api*Result` field, so the golden/completeness + tests are unaffected. +- **Live parity gate (acceptance):** for a viewport + representative filters, the DNF result after + `secondaryFilter` must **equal** the match-all result after `secondaryFilter` (same markers). Since + `secondaryFilter` runs both ways, any divergence is a DNF **under-return** bug. Exercise: a rare quest + reward, a raid boss, an invasion type, a battle pokemon, `onlyGmaxStationed`, `onlyInactiveStations`, + and a mixed filter+match-all-toggle (poisoning) case. + +## 8. Non-goals / deferred + +- **Gym `ex_raid_eligible` / `in_battle` DNF** and **raid/battle/invasion gender & invasion-confirmed + DNF** — stay `secondaryFilter` residual. Fill later only if the observability log shows they cost real + over-fetch. (Gender & quest title/target are structurally inexpressible in DNF and stay residual + permanently.) +- **Station `onlyInactiveStations` / active-vs-inactive / `onlyIncludeUpcoming`** — now-relative + time-window predicates; stay `secondaryFilter` residual permanently (DNF has no `now` concept for + these). +- **Gym badges (`onlyGymBadges`/`onlyBadge`)** — badge gyms surface via a ReactMap-local badge join + (`secondaryFilter` OR); Golbat can't know badge IDs. The gym backend **poisons to `[]`** when either + is active. Fill later only via a badge-id clause if worth it. +- **Pokestop rocket-reward `a` keys** — `invasionMatchesFilters` matches UNCONFIRMED invasions + by the grunt type's *possible* encounters (`info.encounters`), not the confirmed slot, so Golbat's + confirmed-slot `incident_pokemon` would under-return. The pure backend has no reward→grunt map, so it + **poisons to `[]`** on any `a` key. Fill later by threading the event invasion config into the backend + (then emit the matching `incident_character` set) if the log shows it costs real over-fetch. +- **Combined `/api/fort/scan` + `fort_types` scope** — separate optimization, already deferred in the + fort-scan spec §10; sequence after DNF. +- **`is_ar_scan_eligible:false` no-op** in Golbat — not a blocker (ReactMap only sends `true`). + +## 9. Sequencing & packaging + +1. **Golbat station `stationed_gmax` field** (§5) → PR #385. Unblocks the station gmax narrow. +2. **ReactMap backends** (§4) → PR #1228, one per type with review/test checkpoints: **gym** (simplest — + team/raid), then **pokestop** (richest — quest/invasion/showcase), then **station** (battle + the two + new gap fields). The observability log (§6) lands with each. + +Each backend swap is independently reviewable and correctness-safe (secondaryFilter unchanged), so the +slices can merge incrementally. + +## 10. Decisions + +| # | Decision | +|---|---| +| A | **Scope = MVP + station gmax.** Ship the three backends on Golbat's existing DNF fields; fill only `stationed_gmax` in Golbat (a clean column test). Gym ex/in-battle, gender, invasion-confirmed, quest title/target, and station time-window gates (inactive/upcoming) stay residual. | +| B | **DNF is a superset narrow; `secondaryFilter` stays and finalizes.** Correctness can't regress from an imperfect translation; only under-return is a bug. | +| C | **Poisoning rule:** any active match-all toggle or gap category ⇒ backend returns `[]` (match-all). | +| D | **Three pure per-type backend modules** under `server/src/filters/fort/`, not one class. | +| E | **Observability log** shows examined → DNF-dropped → returned → residual-dropped → final, to expose the DNF gap per query. | +| F | Both changes extend the **open PRs** (#385 Golbat, #1228 ReactMap); no new PRs. | diff --git a/docs/superpowers/specs/2026-07-16-fort-scan-map-data-design.md b/docs/superpowers/specs/2026-07-16-fort-scan-map-data-design.md new file mode 100644 index 00000000..0f67edb3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-16-fort-scan-map-data-design.md @@ -0,0 +1,352 @@ +# Golbat Fort DNF Scan → ReactMap Map-Data — Design Spec + +- **Date:** 2026-07-16 +- **Status:** Approved design → planning +- **Golbat branch:** off `feat/pokestop-available-api` (worktree `~/GolandProjects/Golbat-wt/pokestop-available-api`) once that lands; scan infra is pre-existing WIP on that lineage +- **ReactMap branch:** to be cut off `develop` +- **Author:** James Berry (with Claude) +- **Follows:** `2026-07-14-pokestop-available-api-design.md` (this is its "Phase 2") + +## 1. Problem + +ReactMap renders pokestop/gym/raid **map markers, popups, and search** from direct SQL against the +scanner DB (`Pokestop.getAll` `models/Pokestop.js:188`, `Gym.getAll` `models/Gym.js:114`). `getAll` +runs on **every map pan/zoom** — the highest-frequency fort query ReactMap issues. Golbat already +holds every fort in memory (`pokestopCache`/`gymCache`, whole record) and already exposes a +**DNF fort-scan** over a lightweight spatial index. Routing `getAll` through that scan offloads the +per-pan SQL and, with the DNF filter, lets Golbat do the marker narrowing instead of ReactMap. + +The available-list (Phase 1, shipped) already moved to `GET /api/pokestop/available`; this moves the +map-data itself. + +## 2. What already exists (Golbat, pre-existing WIP on this lineage) + +Four **draft, `FortInMemory`-gated** scan endpoints, built on the pokemon-scan template +(`/api/pokemon/v3/scan`): + +| Endpoint | Handler | Response builder | Completeness | +|---|---|---|---| +| `POST /api/gym/scan` | `GymScanEndpoint` | `buildGymResult` → `ApiGymResult` | **complete** (raid + gym detail) | +| `POST /api/pokestop/scan` | `PokestopScanEndpoint` | `buildPokestopResult` → `ApiPokestopResult` | complete **except invasions** | +| `POST /api/station/scan` | `StationScanEndpoint` | `BuildStationResult` → `ApiStationResult` | **complete** (name/battle/stationed/`Battles[]`) | +| `POST /api/fort/scan` | `FortCombinedScanEndpoint` | — | all three in one rtree pass | + +Plus single-record fetch: `GET /api/pokestop/id/{fort_id}` (`routes_huma.go:513`) and +`GET /api/gym/id/{gym_id}` (`routes_huma.go:487`), returning the full `BuildPokestopResult`/ +`BuildGymResult`. **No station by-id endpoint exists** — added by this work (§7.4). + +**Two-phase scan** (`api_fort.go:319-392`): (1) DNF-filter candidate fort ids against the lightweight +value-stored `FortLookup` rtree snapshot via `isFortDnfMatch`; (2) load each matched full record from +the in-memory record cache (`pokestopCache`/`gymCache`, resident when `FortInMemory` is on) and run +the `build*Result` builder. **`FortLookup` is only the filter index; it is never the response +payload.** Request shape mirrors pokemon: `{min, max, limit, dnfFilters[]}`, OR across clauses / AND +within a clause. + +`ApiFortDnfFilter` already carries: shared `PowerUpLevel`/`IsArScanEligible`; gym `AvailableSlots`, +`TeamId[]`, `RaidLevel[]`, `RaidPokemonId[]`; pokestop `LureId[]`, `QuestRewardType[]`, +`QuestRewardAmount`, `QuestRewardItemId[]`, `QuestRewardPokemon[]`, `IncidentDisplayType[]`, +`IncidentStyle[]`, `IncidentCharacter[]`, `IncidentPokemon[]`, `ContestPokemon[]`, +`ContestPokemonType[]`, `ContestTotalEntries`. + +## 3. Goals / Non-Goals + +**Goals** +- Route **`getAll` (markers) and `getOne` (single record)** for **pokestops, gyms/raids, and + stations** through the fort scan / by-id endpoints when a source has a Golbat `endpoint`, mirroring + `Pokemon.getAll`'s `mem` branch. DB fallback (dual source) for un-migrated queries and for + `FortInMemory` off / 503. +- **One per-record mapper per fort type, shared by `getAll` and `getOne`.** The by-id endpoints + return the same struct as one scan element (`ApiPokestopResult`/`ApiGymResult`/`ApiStationResult`), + so `getAll` maps the array and `getOne` maps the single with the same code. +- **Value phase is DNF**: Golbat narrows to matching forts so ReactMap ships a fraction of the forts + and stops running `getAll`'s SQL filter logic. Match-all is only a correctness stepping stone. +- Fill the Golbat gaps: **invasions in the pokestop scan response** (whole `IncidentData` rows fetched + from `incidentCache` via a **string** fetch handle on `FortLookupIncident`; the `int64` re-key is + parked → [UnownHash/Golbat#384]), a **station by-id endpoint** (`GET /api/station/id/{id}`) for + station `getOne`, and **station preload under bare `fort_in_memory`** so the station index is + complete without the full `Preload` config. +- **Gym + station `getAvailable`** (filter lists) — new `GET /api/gym/available` + + `GET /api/station/available`, each a single `fortLookupCache.Range` aggregate mirroring + `/api/pokestop/available`. Unlike pokestops, **no maintained map** is needed: `FortLookup` omits no + gym/station filter field (the pokestop map existed only for quest title/target). +- Keep `FortLookup` DNF-only; consumers receive **whole records from the record caches**, as if + loaded from the DB, so new columns appear automatically. + +**Non-Goals** +- `search`/`getSubmissions`/`getBadges` migration — left on the DB fallback (low frequency; text + search / PoI cells / ReactMap-local badge table). +- Fort-id key-representation change (§10, deferred). +- Any DB **schema** change (incident id stays `varchar` on disk). + +## 4. Decisions (locked) + +| # | Decision | +|---|---| +| D1 | Migrate `getAll` (markers) **and `getOne`** for **pokestops, gyms/raids, and stations** to the fort scan / by-id endpoints — `getOne` reusing each type's per-record `getAll` mapper. `search`/`submissions`/`badges` stay on the bound DB (dual source). | +| D2 | `FortLookup` stays **DNF-only** (filter fields the scan reads). Response = **whole records** from `pokestopCache`/`gymCache`/`incidentCache`, like a DB load — future-proof to new columns. | +| D3 | Incidents in the pokestop scan via **Option A**: an `Id` fetch handle on `FortLookupIncident`; whole `IncidentData` row loaded from `incidentCache`. Gated by a **`with_incidents`** request flag (default off; ReactMap sets it when the invasion layer is active). Read-through to DB on the rare cache miss. | +| D4 | Incident fetch handle (D3) is a plain **`string`**: `incidentCache` stays string-keyed; `FortLookupIncident.Id string` copied from `incident.Id`; `incidentCache.Get(id)` directly — no new type, no parse. The `int64` re-key (native-int key via `Int64Str`) is **parked** → [UnownHash/Golbat#384]. Rationale: the incident id is a proto *string* (`PokestopIncidentDisplayProto.IncidentId`, set with no parse), so a re-key needs a fallible parse-on-ingest with a silent-drop mode; the feature doesn't need it. Ids are numeric on the validating instance (1091/1091), but the string handle carries zero portability risk. | +| D5 | Filter: **match-all MVP** (Golbat returns all forts in bbox; ReactMap's existing `secondaryFilter` unchanged) → **DNF immediately after** (translate ReactMap's filter to `ApiFortDnfFilter`). Back-to-back, DNF is the deliverable. | +| D6 | **Gyms first** — zero Golbat change (response complete, DNF fields exist), so both match-all and DNF land end-to-end with no Golbat work. Pokestops second (need D3). | +| D7 | Area restriction moves from SQL `ST_CONTAINS` to app-side `filterRTree` (already wired for `Pokemon`, add for forts). | +| D8 | Dual-source `getAll`: `mem` branch → scan endpoint, else/​on-503 → bound DB. `deDupeResults` keeps the larger `updated`, so the endpoint must return `updated` in **unix seconds** matching SQL. | +| D9 | Fort-id key representation stays **string** (128-bit hex + `.NN` suffix; can't collapse to `uint64`). Deferred, profile-first (§10). | +| D10 | 25h resident-cache eviction gap **accepted** (raise the TTL if it ever bites); the bound DB source is also a completeness floor. | +| D11 | Ship as **one Golbat PR** (G1+G2) + **one ReactMap PR** (gyms+pokestops, match-all+DNF), each built whole but with **review/test checkpoints** at intermediate states. `with_incidents` = body field on `ApiFortScan`; DNF via a fort filter `Backend` mirroring `PkmnBackend`. | + +## 5. Id analysis (drove D4, D9) + +- **Fort ids (pokestop/gym):** e.g. `85e40e3a838b41a08589eb19fc35611b.16` — 32 hex chars (**128-bit**) + + a `.NN` byte-ish suffix = 35 chars, exactly `varchar(35)`. Wider than `uint64`; a fixed + `[16]byte`+tag key is the only non-lossy option and only sheds string overhead on a 128-bit hash — + marginal. **Stays string** (D9). +- **Incident ids:** e.g. `-1016089077232382347` — numeric (signed int64) as decimal-string values in + `varchar(35)` on the validating instance (1091/1091). But the proto field + (`PokestopIncidentDisplayProto.IncidentId`) is a **`string`** set with no parse — unlike pokemon's + `EncounterId uint64` — so an `int64` re-key needs a fallible parse-on-ingest (silent-drop mode for a + non-numeric id). The feature doesn't need it, so this PR uses a **string** fetch handle (D4) and the + int64 re-key is parked → [UnownHash/Golbat#384]. + +## 6. Incident handle: string (int64 re-key parked → #384) + +This PR adds the fetch handle as a plain `string` (`FortLookupIncident.Id string`, copied from +`incident.Id`; `incidentCache.Get(id)` directly) — no new type, no re-key, no parse. The `int64` +optimization (a signed `Int64Str` mirroring `decoder/uint64str.go` + an `int64`-keyed `incidentCache`, +the way `pokemonCache` keys pokemon by `uint64`) is recorded in **[UnownHash/Golbat#384]** with its +prerequisites (universal-numeric confirmation + a profile). Not in scope here. + +## 7. Golbat design + +### 7.1 Incident fetch handle — string (D4) +- `FortLookupIncident` gains `Id string` (`station_battle.go:51`), copied in + `updatePokestopIncidentLookup` (`fortRtree.go:270`) from `incident.Id` (the full `*Incident` is + already in hand) — rides the existing atomic `Compute`, no new structure/lock ordering, no re-key. +- No `Int64Str`, no `incidentCache` re-key, no parse. (The `int64` optimization is parked → #384.) + +### 7.2 Invasions in the pokestop scan (D3) +- `ApiPokestopResult` gains `Invasions []ApiPokestopIncident` (whole-row shape: character/display_type/ + style/confirmed/slots/expiration/etc.). Populated **only when `with_incidents`** is set: for each + matched fort, `fortLookupCache.Load(fortId)` (the scan's phase-2 loop has only the fort-id string, + not the `FortLookup` — `api_fort.go:349`), iterate its `Incidents`, `incidentCache.Get(inc.Id)` → + whole `IncidentData` → map. `getOne` by-id does the same `fortLookupCache.Load(fortId)`. +- **Miss path:** incident lifetime (~30–60 min) ≪ cache TTL (~25h), so an active incident is + essentially always resident; the `Get` miss is a rare race → read through `getIncidentRecordReadOnly` + (existing DB-load) to preserve the whole-row contract. Expired entries skipped by `ExpireTimestamp`. +- `WithIncidents bool` added to the shared **`ApiFortScan` request body** (`json:"with_incidents"`, + `required:"false"`, default false) — matches the scan endpoints' all-fields-in-POST-body pattern. + Honored by the pokestop and combined `/fort/scan` handlers; gym/station ignore it. (By-id `getOne` + attaches incidents unconditionally, or gains its own query flag if payload matters.) + +### 7.3 No `FortLookup` display-widening +`FortLookup` keeps only what `isFortDnfMatch` reads (incl. the slot1 incident projection it filters +on). The id in 7.2 is a **record locator**, not display data. Whole invasion rows come from +`incidentCache`, whole pokestop rows from `pokestopCache`, whole gym rows from `gymCache`. + +### 7.4 Station by-id endpoint (new) — for station `getOne` +`GET /api/station/id/{station_id}` → `BuildStationResult`, mirroring the existing gym/pokestop by-id +routes (`routes_huma.go:487,513`): `FortInMemory`-gated, `Security: golbatSecret`, read-through to DB +on cache miss. Trivial; the only Golbat addition stations need (their scan response is already +complete). Pokestop/gym by-id already exist; the pokestop by-id should attach incidents (via +`fortLookupCache.Load(fortId)`) so `getOne` popups carry invasions — for a single record, attach them +unconditionally rather than behind `with_incidents`. + +### 7.5 Station preload under bare `fort_in_memory` (trivial) +`PreloadForts` (`preload.go:63`, the bare-`fort_in_memory` path) loads pokestops (`:106`) and gyms +(`:171`) but **not** stations; only the full `Preload` (`:16`) calls `preloadStations` (`:211`) + +`preloadStationBattles` (`:48`). Add those two existing calls to `PreloadForts` so a +`fort_in_memory`-only instance indexes stations (and their battles) completely — otherwise station +scans undercount until each station is next touched. ~2 lines; both functions already exist. + +### 7.6 Gym + station `getAvailable` (new endpoints — pure scans) +`GET /api/gym/available` and `GET /api/station/available`, each a single `fortLookupCache.Range` over +`FortType == GYM`/`STATION`, structured like `GetAvailablePokestops` (`api_pokestop_available.go`) but +**without** the maintained-map machinery: +- **Gym** — per resident gym emit team/slots (`TeamId`,`AvailableSlots`); and if `RaidEndTimestamp > + now && RaidLevel > 0`, the raid level, plus a boss `(RaidPokemonId,RaidPokemonForm)` when + `RaidPokemonId != 0` else an egg at `RaidLevel`. Structured tuples + counts; ReactMap builds + `t`/`g`/`r`/`e`/boss keys. +- **Station** — per resident station iterate `StationBattles` (or the top battle), `BattleEndTimestamp + > now`, emit `(BattlePokemonId,BattlePokemonForm,BattleLevel)`. ReactMap builds `-`/`j` + keys. +- No `FortLookup` change, no reconcile, no cross-check — every field is already present and DNF-used. + `FortInMemory`-gated, `Security: golbatSecret`, instrumented like §6. + +## 8. ReactMap output contract to reproduce + +`Pokestop.getAll`/`Gym.getAll` return marker objects consumed by the GraphQL resolvers and the map. +The mapper must reproduce the SQL path's fields (perms-gated in `secondaryFilter`, which stays as-is): + +**Pokestop** — core `id,lat,lon,enabled,url,name,last_modified_timestamp,updated`; `ar_scan_eligible, +power_up_points,power_up_level,power_up_end_timestamp`; `lure_id,lure_expire_timestamp`; `quests[]` +(both AR/no-AR layers, reward-type-specific fields + `title`); `invasions[]` (`grunt_type,display_type, +confirmed,incident_expire_timestamp,slot_1_*`, slots 2/3 null); `events[]` (showcase fields). +Filter-key vocabulary (for DNF): `l`, `q/d/u/p/c/x/m`(+pokémon), `i/a/b`, `f/h`. + +**Gym** — core `id,name,url,lat,lon,updated,last_modified_timestamp`; gym `team_id,available_slots, +ex_raid_eligible,ar_scan_eligible,in_battle,guarding_pokemon_id,guarding_pokemon_display,defenders, +total_cp,power_up_*`; raid `raid_level,raid_battle_timestamp,raid_end_timestamp,raid_pokemon_id/form/ +gender/costume/evolution/move_1/move_2/alignment`; computed `hasRaid`/`hasGym`. Filter vocabulary: +`e`(egg tier), `t`(team), `g`(team-slots), raid-boss `-`(+gender). `getBadges` stays on the +ReactMap-local DB (out of scope). + +**Station** (`Station.getAll` `models/Station.js:588`) — core `id,name,lat,lon,updated`; battle +`start_time,end_time,is_battle_available,battle_level,battle_pokemon_*`(id/form/costume/gender/ +alignment/bread_mode/move_1/move_2), `total_stationed_pokemon,total_stationed_gmax,stationed_pokemon`, +`battles[]`. Filter vocabulary: `onlyMaxBattles`, `onlyBattleTier`, `onlyGmaxStationed`, +`onlyIncludeUpcoming` → DNF `BattleLevel[]`/`BattlePokemon[]`. `ApiStationResult` already carries all +of this. + +**`getOne`** (all three types): the by-id endpoint returns one `Api*Result`; run it through the same +per-record mapper `getAll` uses, then return it. ReactMap's `getOne` today yields only `lat,lon` for +recenter — the endpoint superset is harmless. + +Invasion mapping from the whole incident row: `Character→grunt_type`, `DisplayType→display_type`, +`Confirmed→confirmed`, `ExpirationTime→incident_expire_timestamp`, `Slot1PokemonId→slot_1_pokemon_id`, +`Slot1Form→slot_1_form` (covers kecleon 8 / goldstop 7 blocker display types too). + +## 9. ReactMap design + +**Endpoint response shapes — read these, don't assume a bare array.** The fort *scan* endpoints +return an **envelope**, NOT a bare array like `/api/pokemon/v2/scan` (which the `Pokemon.getAll` +template mirrors). A `getAll` `mem` branch must read the typed array off the envelope, or +`Array.isArray(res)` is always false and it silently falls back to SQL on a healthy 200 (the bug that +bit the gyms slice — fixed by reading `res.gyms`): + +| Endpoint | Response | `getAll`/consumer reads | +|---|---|---| +| `POST /api/gym/scan` | `{ gyms:[], examined, skipped, total }` | `res.gyms` | +| `POST /api/pokestop/scan` | `{ pokestops:[], examined, skipped, total }` | `res.pokestops` | +| `POST /api/station/scan` | `{ stations:[], examined, skipped, total }` | `res.stations` | +| `POST /api/fort/scan` (combined) | `{ gyms:[], pokestops:[], stations:[], examined, skipped, total }` | per type | +| `GET /api/{gym\|pokestop\|station}/id/{id}` | bare `Api*Result` object | check `res.lat`/`res.lon` | +| `GET /api/gym/available` | `{ teams:[], raids:[] }` | `res.teams`/`res.raids` | +| `GET /api/pokestop/available` | `{ quests:[], invasions:[], lures:[], showcases:[] }` | those arrays | +| `GET /api/station/available` | `{ battles:[] }` | `res.battles` | + +Diagnose fallbacks with a shared `describeScannerResponse(res)` (HTTP status / body shape / network), +not a hard-coded "fort_in_memory off" guess. + +Mirror `Pokemon.getAll` (`models/Pokemon.js:131`, `mem` branch), for `Pokestop`/`Gym`/`Station`: + +- **`getAll` `mem` branch**: `POST {mem}/api/{gym|pokestop|station}/scan` with + `{min,max,limit,filters,with_incidents?}` via `evalQuery` (sets `X-Golbat-Secret`/httpAuth); read the + matched array off the **envelope** (`res.{gyms|pokestops|stations}`, per the table above). On 503 + / network error / any non-envelope response → fall through to the SQL block (dual source: bound DB + runs it; pure-endpoint source drops out of `Promise.allSettled`). Plumbing from the Phase-1 available + PR is reused. +- **`getOne` `mem` branch**: `GET {mem}/api/{gym|pokestop|station}/id/{id}` → the **same per-record + mapper**; SQL fallback otherwise. +- **Pure per-record mappers** `mapGymResult` / `mapPokestopResult` / `mapStationResult` (like + `pokestopAvailableMapper.js`), each projecting one whole-record `Api*Result` into the marker shape + `secondaryFilter` expects — shared by that type's `getAll` (map the array) and `getOne` (map the + single). `secondaryFilter` (perms) is source-agnostic and stays unchanged. +- **Area restriction (D7):** post-filter results through `filterRTree` (Golbat can't know ReactMap's + area polygons), as `Pokemon.getAll`/`.search` already do. New wiring for forts. +- **`updated` (D8):** ensure the mapper carries `updated` as unix seconds so `deDupeResults` behaves in + dual DB+endpoint mode. +- **Filter (D5):** + - *Phase 1 — match-all:* send an empty/permissive filter, let `secondaryFilter` do all matching in + JS. Correct by construction; unchanged filter logic. Caveat: returns all bbox forts (dense-city + volume). + - *Phase 2 — DNF:* a **fort filter `Backend`** with `buildApiFilter()`, mirroring + `server/src/filters/pokemon/Backend.js` (`PkmnBackend`), translates ReactMap's flat category + filter to `ApiFortDnfFilter[]` (one disjunct per active category). Sub-filters `FortLookup` can't + express (quest **title/target**) stay JS post-filters via `secondaryFilter`, as in Phase 1's + available work. This is the payoff phase. + +## 10. Deferred / follow-ups + +- **Fort-id `[16]byte` key (D9):** profile fort-id hashing first (likely dominated by the rtree walk + + DNF compares); only worth it if measured, and it touches every fort cache + rtree + both boundaries + with an outlier-validation burden. What the `.NN` suffix means (S2 level? source tag? — determines + whether it's in the key) is an open input. +- **Incident `int64` re-key → [UnownHash/Golbat#384]** — parked. `Int64Str` + `int64`-keyed + `incidentCache` + `int64` handle. Prereqs: confirm incident ids are numeric across all deployments + (the proto field is a string), and profile the win. This PR uses a string handle instead. +- `search`/`getSubmissions`/`getBadges` stay on the DB (§3 non-goals) — migrate later only if the + per-record SQL they issue proves worth it. +- **Combined `/api/fort/scan` via a request-scoped batch (optimization).** ReactMap currently hits + `/api/{gym,pokestop,station}/scan` once **per enabled fort layer** (each an independent GraphQL + resolver → `SubModel.getAll`), so a 3-layer view makes 3 separate rtree passes + round-trips. The + combined `POST /api/fort/scan` returns `{gyms,pokestops,stations,examined,skipped,total}` in one + rtree pass. To adopt it: add a **request-scoped memoized fetch** keyed by `(bbox, source)` — the + first model's `getAll` triggers `/api/fort/scan`, the others read the cached envelope and take their + slice. To avoid over-fetching layers that are off, add an **overall scope field to `ApiFortScan`** + (e.g. `fort_types: ["gym","station"]`, honored by `FortCombinedScanEndpoint`) so the combined scan + only processes/returns the enabled types — cleaner than trying to suppress a whole type via DNF + clauses. Best sequenced **after DNF** (so the per-type `ApiFortDnfFilter[]` union is settled). Golbat + side = the `fort_types` scope field; ReactMap side = the batch/dedupe layer. +- **`onlyManualId` out-of-viewport (cross-model).** The `getAll` `mem` branches (gym, station, and + future pokestop) send only the bbox, so a manually-targeted deep-link station/gym *outside* the + viewport isn't returned (the SQL path pulls it via `... OR id = manualId`). Mirror `Pokemon.getAll`'s + manual-id fallback (a `GET /api/{type}/id/{id}` when the bbox scan didn't include the pinned id). + Shared across all fort `mem` models — do once. + +## 11. Packaging & build order (D11) + +**Two PRs**, each built whole but with **pause-for-review/test checkpoints** at intermediate states. + +**Golbat PR** = §7.1–§7.6: incident **string** fetch handle on `FortLookupIncident`; `with_incidents` ++ `Invasions[]` on the pokestop scan (+ pokestop by-id incidents); `GET /api/station/id/{id}`; station +preload under `fort_in_memory`; `GET /api/gym/available` + `GET /api/station/available` (pure scans). +Standalone-testable via `curl` on the scan / by-id / available routes. Ships and deploys **first** — +it's the dependency for the pokestop `getAll` half of the ReactMap PR (gyms/stations `getAll`, all +`getOne`-by-id, and gym/station `getAvailable` don't need it, but a single deploy is simplest). + +**ReactMap PR** = the full consumer (pokestops + gyms + stations; `getAll` match-all → DNF; `getOne`; +gym/station `getAvailable`), built match-all-first so a cross-fort-type state is testable early +(matches the "move quickly past match-all" intent): + +| Step | Content | Dep | Checkpoint | +|---|---|---|---| +| 1 | **Gyms** `getAll` mem branch + per-record mapper + `filterRTree` (match-all) + `getOne` + `getAvailable` | — | test gyms | +| 2 | **Stations** `getAll` + mapper + `getOne` + `getAvailable` (match-all) | — | | +| 3 | **Pokestops** `getAll` (+`with_incidents`) + mapper + `getOne` (match-all) | Golbat PR | **test all three, match-all** | +| 4 | **Gyms / Stations / Pokestops** DNF (fort `Backend`) | 1–3 | **test full DNF** | + +(Gym/station `getAvailable` mirror the shipped `Pokestop.getAvailable` `mem` branch — small, and they +depend only on the two new available endpoints, not on the scan work.) + +Gyms + stations (steps 1–2) carry **no Golbat dependency**, so they prove the whole consumer pattern +(per-record mapper shared by `getAll`/`getOne`, dual source, `filterRTree`, match-all→DNF) even before +the Golbat PR is built. `getOne` rides each type's mapper, so it's folded into that type's step, not a +separate phase. Exact checkpoint grouping is a build-time call; the natural pauses are after all-three +match-all and after full DNF. + +## 12. Coverage caveats + +- **Resident set vs whole DB (D10):** the scan reflects the ~25h-resident fort set; long-idle forts in + unscanned regions can drop until re-touched. Accepted (raise TTL if needed); the bound DB source + remains a floor. +- **`FortInMemory` required** → 503 → SQL fallback (dual source). +- **Stations:** the bare-`fort_in_memory` preload gap is *fixed here* (§7.5), so stations are indexed + from startup like pokestops/gyms — no longer a standing caveat. +- **Gym/station `getAvailable` semantics (§7.6):** the SQL station-available adds `is_inactive=false` + + `updated > activeCutoff` hygiene filters; the scan approximates "active" via `BattleEndTimestamp > + now` + residency (a station with no live battle emits no keys anyway). Gym team/slots have no time + filter (all resident gyms). Both are the same resident-set approximation as D10 — accepted. +- **Base-branch coordination:** §7 touches `fortRtree.go`/`incident_state.go`/`api_fort.go`/`preload.go` + on a branch reworking eviction/locking; keep changes additive; rebase and re-verify accessor names. + +## 13. Testing + +**Golbat** — `FortLookupIncident.Id` carried through `updatePokestopIncidentLookup` (concurrency test +still green). `with_incidents` on/off, multi-incident stop (~11%), whole-row fields incl. slot1, +cache-miss read-through, expired skipped, `getOne` by-id incident attach. Gym/station `available` +aggregates (team/slots/raid/egg; battle level/pokemon incl. multi-battle) with expiry exclusion. +Station by-id returns the record; station preload under `fort_in_memory` indexes stations. Gating: +`!FortInMemory`→503 on the gated routes. + +**ReactMap** — mapper golden vs SQL `getAll` on live data (gyms first, then pokestops incl. invasions), +per perms gate; dual-source fallback (503 → SQL); `filterRTree` area exclusion; DNF phase: assert the +translated filter returns the same marker set as match-all + JS filtering on the same bbox. + +## 14. Resolved (were open) + +1. `with_incidents` — **body field** on `ApiFortScan` (`json:"with_incidents"`, default false), + matching the scan endpoints' all-fields-in-POST-body pattern; honored by pokestop + combined scans. +2. DNF translation — a **fort filter `Backend`** with `buildApiFilter()`, mirroring + `server/src/filters/pokemon/Backend.js` (`PkmnBackend`). +3. Packaging — **one Golbat PR** (§7.1–§7.6) + **one ReactMap PR** (gyms+pokestops+stations, + match-all+DNF), with review/test checkpoints (§11); DNF is built in the same PR, not a follow-up. +4. Incident id typing — **string handle** this PR; `int64` re-key parked → [UnownHash/Golbat#384]. From 9d13b42f1b2a484bc20eabbc39b6aa8a2e885416 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 16:14:33 +0100 Subject: [PATCH 03/29] refactor(dnf): remove unused filter surface; stationed_gmax symmetry; doc clarity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review audit: three DNF fields are structurally unusable by the consumer and had no other reader — incident_style (+ FortLookupIncident.Style), incident_pokemon (consumer expands rocket-reward keys to incident_character instead; slot1 lookup fields stay for /pokestop/available), and contest_total_entries (+ FortLookup.ContestTotalEntries and the per-save showcase-rankings JSON parse). power_up_level/team_id/available_slots stay: secondaryFilter narrows by them in all-gyms mode, so they are future-sendable. stationed_gmax:false now matches gmax-less stations (symmetric with station_active); top-level filters doc clarified (omitted/empty array = match-all; empty inner lists match nothing). Co-Authored-By: Claude Fable 5 --- decoder/api_fort.go | 31 +++++-------------- decoder/fortRtree.go | 30 +++--------------- decoder/fortRtree_compute_test.go | 1 - decoder/fort_incident_test.go | 6 ++-- decoder/station_battle.go | 1 - .../2026-07-16-fort-dnf-filtering-design.md | 19 ++++++++++++ 6 files changed, 34 insertions(+), 54 deletions(-) diff --git a/decoder/api_fort.go b/decoder/api_fort.go index 2b8c496b..e611dd12 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -15,7 +15,7 @@ type ApiFortScan struct { Min ApiLatLon `json:"min" doc:"SW (minimum lat/lon) corner of the bounding box."` Max ApiLatLon `json:"max" doc:"NE (maximum lat/lon) corner of the bounding box."` Limit int `json:"limit" required:"false" doc:"Max results to return; 0 uses the server default."` - DnfFilters []ApiFortDnfFilter `json:"filters" required:"false" doc:"OR'd filter clauses; a fort matches if it satisfies any one clause. List conditions apply only when present: omit or send null for no constraint — an explicitly empty list matches nothing."` + DnfFilters []ApiFortDnfFilter `json:"filters" required:"false" doc:"OR'd filter clauses; a fort matches if it satisfies any one clause. Omitting this array (or sending it empty/null) matches ALL forts of the requested type. Within a clause, a list-typed condition applies only when present: omit or send null for no constraint — an explicitly empty inner list matches nothing."` WithIncidents bool `json:"with_incidents" required:"false" doc:"Pokestop only: when true, each pokestop result includes its active incidents (invasions). Ignored for gym/station."` } @@ -37,20 +37,17 @@ type ApiFortDnfFilter struct { QuestRewardPokemon []ApiDnfId `json:"quest_reward_pokemon" required:"false" doc:"Pokestop only: allowed quest reward pokemon/form pairs; matched against either the AR or no-AR quest. Omitted or null means no reward pokemon constraint."` // Pokestop - incident - IncidentDisplayType []int8 `json:"incident_display_type" required:"false" doc:"Pokestop only: allowed incident display types; omitted or null means no incident display type constraint."` - IncidentStyle []int8 `json:"incident_style" required:"false" doc:"Pokestop only: allowed incident styles; omitted or null means no incident style constraint."` - IncidentCharacter []int16 `json:"incident_character" required:"false" doc:"Pokestop only: allowed incident character ids; omitted or null means no incident character constraint."` - IncidentPokemon []ApiDnfId `json:"incident_pokemon" required:"false" doc:"Pokestop only: allowed incident pokemon/form pairs; omitted or null means no incident pokemon constraint."` + IncidentDisplayType []int8 `json:"incident_display_type" required:"false" doc:"Pokestop only: allowed incident display types; omitted or null means no incident display type constraint."` + IncidentCharacter []int16 `json:"incident_character" required:"false" doc:"Pokestop only: allowed incident character ids; omitted or null means no incident character constraint."` // Pokestop - contest - ContestPokemon []ApiDnfId `json:"contest_pokemon" required:"false" doc:"Pokestop only: allowed contest focus pokemon/form pairs; omitted or null means no contest pokemon constraint."` - ContestPokemonType []int8 `json:"contest_pokemon_type" required:"false" doc:"Pokestop only: allowed contest pokemon types; omitted or null means no contest type constraint."` - ContestTotalEntries *ApiFortDnfMinMax `json:"contest_total_entries" required:"false" doc:"Pokestop only: inclusive range for the contest's total number of entries; null means no contest entries constraint."` + ContestPokemon []ApiDnfId `json:"contest_pokemon" required:"false" doc:"Pokestop only: allowed contest focus pokemon/form pairs; omitted or null means no contest pokemon constraint."` + ContestPokemonType []int8 `json:"contest_pokemon_type" required:"false" doc:"Pokestop only: allowed contest pokemon types; omitted or null means no contest type constraint."` // Station BattleLevel []int8 `json:"battle_level" required:"false" doc:"Station only: allowed active max battle levels; omitted or null means no battle level constraint. Only matches stations with an active battle."` BattlePokemon []ApiDnfId `json:"battle_pokemon" required:"false" doc:"Station only: allowed active max battle pokemon/form pairs; omitted or null means no battle pokemon constraint. Only matches stations with an active battle."` - StationedGmax *bool `json:"stationed_gmax" required:"false" doc:"Station only: when true, only match stations with at least one stationed Gigantamax pokemon; null means no constraint."` + StationedGmax *bool `json:"stationed_gmax" required:"false" doc:"Station only: when true, only match stations with at least one stationed Gigantamax pokemon; when false, only stations without any. Null means no constraint."` StationActive *bool `json:"station_active" required:"false" doc:"Station only: when true, only match stations whose end_time is in the future (still present); when false, only expired stations. Stations are the one ephemeral fort type — expired ones accumulate in the index. Null means no constraint."` } @@ -178,19 +175,13 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn (fortLookup.ShowcaseExpiry <= now || !slices.Contains(filter.ContestPokemonType, fortLookup.ContestPokemonType)) { return false } - if filter.ContestTotalEntries != nil && - (fortLookup.ShowcaseExpiry <= now || - fortLookup.ContestTotalEntries < filter.ContestTotalEntries.Min || fortLookup.ContestTotalEntries > filter.ContestTotalEntries.Max) { - return false - } if filter.ContestPokemon != nil && (fortLookup.ShowcaseExpiry <= now || !matchDnfIdPair(filter.ContestPokemon, fortLookup.ContestPokemonId, fortLookup.ContestPokemonForm)) { return false } // Incident filters - match any non-expired incident in the slice - if filter.IncidentDisplayType != nil || filter.IncidentStyle != nil || - filter.IncidentCharacter != nil || filter.IncidentPokemon != nil { + if filter.IncidentDisplayType != nil || filter.IncidentCharacter != nil { matched := false for _, inc := range fortLookup.Incidents { if inc.ExpireTimestamp <= now { @@ -199,15 +190,9 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn if filter.IncidentDisplayType != nil && !slices.Contains(filter.IncidentDisplayType, inc.DisplayType) { continue } - if filter.IncidentStyle != nil && !slices.Contains(filter.IncidentStyle, inc.Style) { - continue - } if filter.IncidentCharacter != nil && !slices.Contains(filter.IncidentCharacter, inc.Character) { continue } - if filter.IncidentPokemon != nil && !matchDnfIdPair(filter.IncidentPokemon, inc.Slot1PokemonId, inc.Slot1Form) { - continue - } matched = true break } @@ -219,7 +204,7 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn if filter.StationActive != nil && *filter.StationActive != (fortLookup.StationEndTimestamp > now) { return false } - if filter.StationedGmax != nil && *filter.StationedGmax && fortLookup.TotalStationedGmax <= 0 { + if filter.StationedGmax != nil && *filter.StationedGmax != (fortLookup.TotalStationedGmax > 0) { return false } if filter.BattleLevel != nil || filter.BattlePokemon != nil { diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 7e02ba55..4001f5bd 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -1,12 +1,10 @@ package decoder import ( - "encoding/json" "sync" "sync/atomic" "time" - "github.com/guregu/null/v6" "github.com/puzpuzpuz/xsync/v4" log "github.com/sirupsen/logrus" "github.com/tidwall/rtree" @@ -51,11 +49,10 @@ type FortLookup struct { Incidents []FortLookupIncident // Pokestop - contest - ContestPokemonId int16 - ContestPokemonForm int16 - ContestPokemonType int8 - ContestTotalEntries int16 - ShowcaseExpiry int64 // used to check expiry at filter time + ContestPokemonId int16 + ContestPokemonForm int16 + ContestPokemonType int8 + ShowcaseExpiry int64 // used to check expiry at filter time // Station StationEndTimestamp int64 // station end_time; liveness gate at filter time @@ -193,7 +190,6 @@ func updatePokestopLookup(pokestop *Pokestop) { // each preserving the other's fields. A plain Load->Store pair can // interleave and clobber. Keep the callback to field copies — the // showcase-rankings JSON parse is hoisted out. - contestTotalEntries := getContestTotalEntries(pokestop.ShowcaseRankings) fortLookupCache.Compute(pokestop.Id, func(existing FortLookup, loaded bool) (FortLookup, xsync.ComputeOp) { nl := FortLookup{ FortType: POKESTOP, @@ -216,7 +212,6 @@ func updatePokestopLookup(pokestop *Pokestop) { ContestPokemonId: int16(pokestop.ShowcasePokemon.ValueOrZero()), ContestPokemonForm: int16(pokestop.ShowcasePokemonForm.ValueOrZero()), ContestPokemonType: int8(pokestop.ShowcasePokemonType.ValueOrZero()), - ContestTotalEntries: contestTotalEntries, ShowcaseExpiry: pokestop.ShowcaseExpiry.ValueOrZero(), } if loaded { @@ -276,7 +271,6 @@ func updatePokestopIncidentLookup(pokestopId string, incident *Incident) { updated := FortLookupIncident{ Id: incident.Id, DisplayType: int8(incident.DisplayType), - Style: int8(incident.Style), Character: incident.Character, Confirmed: incident.Confirmed, Slot1PokemonId: int16(incident.Slot1PokemonId.ValueOrZero()), @@ -310,22 +304,6 @@ func updatePokestopIncidentLookup(pokestopId string, incident *Incident) { }) } -// getContestTotalEntries parses showcase rankings JSON to get total entries -func getContestTotalEntries(rankingsString null.String) int16 { - if !rankingsString.Valid { - return -1 - } - - type contestJson struct { - TotalEntries int `json:"total_entries"` - } - var cj contestJson - if json.Unmarshal([]byte(rankingsString.String), &cj) == nil { - return int16(cj.TotalEntries) - } - return -1 -} - func addFortToTree(id string, lat float64, lon float64) { fortTreeMutex.Lock() fortTree.Insert([2]float64{lon, lat}, [2]float64{lon, lat}, id) diff --git a/decoder/fortRtree_compute_test.go b/decoder/fortRtree_compute_test.go index 2b63c58c..2b696f03 100644 --- a/decoder/fortRtree_compute_test.go +++ b/decoder/fortRtree_compute_test.go @@ -26,7 +26,6 @@ func TestFortLookupConcurrentPokestopAndIncidentWriters(t *testing.T) { }} inc := &Incident{IncidentData: IncidentData{ DisplayType: 3, - Style: 2, Character: 44, Slot1PokemonId: null.IntFrom(215), ExpirationTime: 9_999_999_999, // far future so the upsert does not prune it diff --git a/decoder/fort_incident_test.go b/decoder/fort_incident_test.go index 8f81658b..f7ecaf35 100644 --- a/decoder/fort_incident_test.go +++ b/decoder/fort_incident_test.go @@ -17,9 +17,9 @@ func TestFortDnfMatch_IncidentSlice(t *testing.T) { if !isFortDnfMatch(POKESTOP, fl, &ApiFortDnfFilter{IncidentDisplayType: []int8{9}}, now) { t.Fatal("showcase (dt9) should match") } - // slot1 pokemon matches - if !isFortDnfMatch(POKESTOP, fl, &ApiFortDnfFilter{IncidentPokemon: []ApiDnfId{{Pokemon: 41}}}, now) { - t.Fatal("slot1 pokemon 41 should match") + // active grunt character matches + if !isFortDnfMatch(POKESTOP, fl, &ApiFortDnfFilter{IncidentCharacter: []int16{5}}, now) { + t.Fatal("grunt (5) should match") } // expired incident does not match if isFortDnfMatch(POKESTOP, fl, &ApiFortDnfFilter{IncidentCharacter: []int16{30}}, now) { diff --git a/decoder/station_battle.go b/decoder/station_battle.go index d3f03cf0..23225dab 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -51,7 +51,6 @@ type FortLookupStationBattle struct { type FortLookupIncident struct { Id string // incident id — fetch handle into incidentCache (not DNF-used) DisplayType int8 - Style int8 Character int16 Confirmed bool Slot1PokemonId int16 diff --git a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md index 53fedea7..73de3edb 100644 --- a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md +++ b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md @@ -240,3 +240,22 @@ slices can merge incrementally. | D | **Three pure per-type backend modules** under `server/src/filters/fort/`, not one class. | | E | **Observability log** shows examined → DNF-dropped → returned → residual-dropped → final, to expose the DNF gap per query. | | F | Both changes extend the **open PRs** (#385 Golbat, #1228 ReactMap); no new PRs. | + + +## 11. Post-review dead-surface cleanup (2026-07-17) + +The dual-PR review audited every `ApiFortDnfFilter` field against what consumers send. Removed +(structurally unusable by ReactMap, no other reader): + +- `incident_style` (+ `FortLookupIncident.Style`) — ReactMap has no style concept. +- `incident_pokemon` — deliberately rejected consumer-side (confirmed-slot semantics under-return; + `a` keys expand to `incident_character` instead). `FortLookupIncident.Slot1PokemonId/Form` stay — + `/api/pokestop/available` reads them. +- `contest_total_entries` (+ `FortLookup.ContestTotalEntries` and the per-save showcase-rankings JSON + parse) — no entry-count filter exists in the UI. + +Kept though currently unsent — **future-sendable**: `power_up_level`, `team_id`, `available_slots`. +In `onlyAllGyms` mode `secondaryFilter` DOES narrow by team/slot/power-up keys, so the current +poison-to-match-all for that mode could be replaced by real clauses using these fields (follow-up +optimization). Also from review: `stationed_gmax:false` now symmetric (matches gmax-less stations); +top-level `filters` doc clarified (omitted/empty array = match-all; empty *inner* lists match nothing). From 0914013e12ee3e9a6724e0eab1559a4f5c09a222 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 16:23:11 +0100 Subject: [PATCH 04/29] refactor(dnf): drop power_up_level from the fort filter and lookup Power-ups are no longer in the game: no consumer can meaningfully filter on them, so remove the DNF field, the matcher branch, and the FortLookup byte. The record APIs still expose the power_up_* columns (whole-record principle). Co-Authored-By: Claude Fable 5 --- decoder/api_fort.go | 6 +----- decoder/fortRtree.go | 3 --- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/decoder/api_fort.go b/decoder/api_fort.go index e611dd12..85dd2950 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -20,8 +20,7 @@ type ApiFortScan struct { } type ApiFortDnfFilter struct { - PowerUpLevel *ApiFortDnfMinMax `json:"power_up_level" required:"false" doc:"Inclusive power-up level range; null means no power-up level constraint."` - IsArScanEligible *bool `json:"is_ar_scan_eligible" required:"false" doc:"When true, only match forts that are AR scan eligible; null means no AR eligibility constraint."` + IsArScanEligible *bool `json:"is_ar_scan_eligible" required:"false" doc:"When true, only match forts that are AR scan eligible; null means no AR eligibility constraint."` // Gym AvailableSlots *ApiFortDnfMinMax `json:"available_slots" required:"false" doc:"Gym only: inclusive range of open defender slots; null means no slot constraint."` @@ -108,9 +107,6 @@ func isFortDnfMatch(fortType FortType, fortLookup *FortLookup, filter *ApiFortDn if fortType != 0 && fortType != fortLookup.FortType { return false } - if filter.PowerUpLevel != nil && (int16(fortLookup.PowerUpLevel) < filter.PowerUpLevel.Min || int16(fortLookup.PowerUpLevel) > filter.PowerUpLevel.Max) { - return false - } if filter.IsArScanEligible != nil && !fortLookup.IsArScanEligible { return false } diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 4001f5bd..188ae7fc 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -18,7 +18,6 @@ type FortLookup struct { FortType FortType Lat float64 Lon float64 - PowerUpLevel int8 IsArScanEligible bool // Gym @@ -195,7 +194,6 @@ func updatePokestopLookup(pokestop *Pokestop) { FortType: POKESTOP, Lat: pokestop.Lat, Lon: pokestop.Lon, - PowerUpLevel: int8(valueOrMinus1(pokestop.PowerUpLevel)), IsArScanEligible: pokestop.ArScanEligible.ValueOrZero() == 1, LureId: pokestop.LureId, LureExpireTimestamp: pokestop.LureExpireTimestamp.ValueOrZero(), @@ -233,7 +231,6 @@ func updateGymLookup(gym *Gym) { FortType: GYM, Lat: gym.Lat, Lon: gym.Lon, - PowerUpLevel: int8(valueOrMinus1(gym.PowerUpLevel)), IsArScanEligible: gym.ArScanEligible.ValueOrZero() == 1, AvailableSlots: int8(gym.AvailableSlots.ValueOrZero()), TeamId: int8(gym.TeamId.ValueOrZero()), From 5edf582792c51b4611ad4e79671473a673a36dbd Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 16:25:00 +0100 Subject: [PATCH 05/29] docs(spec): power_up removal + all-gyms team/slot narrowing --- .../specs/2026-07-16-fort-dnf-filtering-design.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md index 73de3edb..711be1d2 100644 --- a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md +++ b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md @@ -259,3 +259,11 @@ In `onlyAllGyms` mode `secondaryFilter` DOES narrow by team/slot/power-up keys, poison-to-match-all for that mode could be replaced by real clauses using these fields (follow-up optimization). Also from review: `stationed_gmax:false` now symmetric (matches gmax-less stations); top-level `filters` doc clarified (omitted/empty array = match-all; empty *inner* lists match nothing). + +**Follow-ups landed (2026-07-17):** `power_up_level` removed entirely from the DNF filter and +`FortLookup` (power-ups are no longer in the game; record APIs still expose the columns). The +all-gyms-mode narrowing landed in ReactMap: the gym layer now sends `team_id`/`available_slots` +clauses mirroring `finalTeams`/`finalSlots` (t/g keys) for ALL four gym-layer enablers +(all-gyms/ex/in-battle/ar) — the previous poison and the standalone `is_ar_scan_eligible` gym clause +are gone; ex/ar/in-battle remain residual halves of an ANDed condition, so the clauses stay a tight +superset. Badge viewing still poisons. From 4c755cfae5a5264551b7ac19a7832ccf75b67e84 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 16:38:09 +0100 Subject: [PATCH 06/29] feat(api): typed clause groups for the combined fort scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The combined /api/fort/scan previously took one flat DnfFilters list applied to every fort type — semantically broken for mixed-type clause sets, because a clause's foreign-type fields are all wildcards: a gym raid clause vacuously matched EVERY pokestop and station. The body is now per-type groups (gyms/pokestops/stations), each carrying its own clause set: - present group with clauses -> OR across them, evaluated only against that type's forts - present group with no clauses -> match-all for that type - omitted/null group -> the type is excluded entirely (this subsumes the planned fort_types scope field) - all three groups omitted -> bare bbox probe, everything matches This is the prerequisite for consumers coalescing their per-type scans into one combined call carrying each type's DNF clauses. Co-Authored-By: Claude Fable 5 --- decoder/api_fort.go | 68 ++++++++++++++++++++++++------- decoder/api_fort_combined_test.go | 50 +++++++++++++++++++++++ routes_huma.go | 4 +- 3 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 decoder/api_fort_combined_test.go diff --git a/decoder/api_fort.go b/decoder/api_fort.go index 85dd2950..0689f38f 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -19,6 +19,56 @@ type ApiFortScan struct { WithIncidents bool `json:"with_incidents" required:"false" doc:"Pokestop only: when true, each pokestop result includes its active incidents (invasions). Ignored for gym/station."` } +// ApiFortTypeScanGroup scopes DNF clauses to ONE fort type within a combined +// scan. Clauses are evaluated only against forts of that type, so a gym +// clause can never vacuously match a pokestop (a clause's foreign-type fields +// are all wildcards for other types). +type ApiFortTypeScanGroup struct { + DnfFilters []ApiFortDnfFilter `json:"filters" required:"false" doc:"OR'd clauses for this fort type; omit, null or empty to match every fort of the type."` +} + +// ApiFortCombinedScan is the request body for the combined /api/fort/scan. +// Each present group opts its fort type into the scan with its own clause +// set; an omitted/null group EXCLUDES that type. Omitting all three groups +// matches every fort of every type (bare bbox probe). +type ApiFortCombinedScan struct { + Min ApiLatLon `json:"min" doc:"SW (minimum lat/lon) corner of the bounding box."` + Max ApiLatLon `json:"max" doc:"NE (maximum lat/lon) corner of the bounding box."` + Limit int `json:"limit" required:"false" doc:"Max results to return across all types; 0 uses the server default."` + WithIncidents bool `json:"with_incidents" required:"false" doc:"When true, each pokestop result includes its active incidents (invasions)."` + Gyms *ApiFortTypeScanGroup `json:"gyms" required:"false" doc:"Include gyms, filtered by this group's clauses. Omitted or null excludes gyms (unless all three groups are omitted)."` + Pokestops *ApiFortTypeScanGroup `json:"pokestops" required:"false" doc:"Include pokestops, filtered by this group's clauses. Omitted or null excludes pokestops (unless all three groups are omitted)."` + Stations *ApiFortTypeScanGroup `json:"stations" required:"false" doc:"Include stations, filtered by this group's clauses. Omitted or null excludes stations (unless all three groups are omitted)."` +} + +// combinedFortMatches applies the typed clause groups to one fort: the fort's +// own type's group governs it exclusively. Excluded type -> false; group with +// no clauses -> match-all for the type; else OR across the group's clauses. +func combinedFortMatches(p *ApiFortCombinedScan, fl *FortLookup, now int64) bool { + var g *ApiFortTypeScanGroup + switch fl.FortType { + case GYM: + g = p.Gyms + case POKESTOP: + g = p.Pokestops + case STATION: + g = p.Stations + } + if g == nil { + // legacy bare-probe: all groups omitted = every type matches + return p.Gyms == nil && p.Pokestops == nil && p.Stations == nil + } + if len(g.DnfFilters) == 0 { + return true + } + for i := range g.DnfFilters { + if isFortDnfMatch(fl.FortType, fl, &g.DnfFilters[i], now) { + return true + } + } + return false +} + type ApiFortDnfFilter struct { IsArScanEligible *bool `json:"is_ar_scan_eligible" required:"false" doc:"When true, only match forts that are AR scan eligible; null means no AR eligibility constraint."` @@ -389,7 +439,7 @@ func StationScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) } } -func FortCombinedScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiFortCombinedScanResult { +func FortCombinedScanEndpoint(retrieveParameters ApiFortCombinedScan, dbDetails db.DbDetails) *ApiFortCombinedScanResult { gymKeys, pokestopKeys, stationKeys, examined, skipped, total := internalGetFortsCombined(retrieveParameters) start := time.Now() now := time.Now().Unix() @@ -450,7 +500,7 @@ func FortCombinedScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDet } } -func internalGetFortsCombined(retrieveParameters ApiFortScan) (gymKeys, pokestopKeys, stationKeys []string, examined, skipped, total int) { +func internalGetFortsCombined(retrieveParameters ApiFortCombinedScan) (gymKeys, pokestopKeys, stationKeys []string, examined, skipped, total int) { start := time.Now() minLocation := retrieveParameters.Min.Location() @@ -480,19 +530,7 @@ func internalGetFortsCombined(retrieveParameters ApiFortScan) (gymKeys, pokestop return true } - matched := false - if len(retrieveParameters.DnfFilters) == 0 { - matched = true - } else { - for i := range retrieveParameters.DnfFilters { - if isFortDnfMatch(0, &fortLookup, &retrieveParameters.DnfFilters[i], now) { - matched = true - break - } - } - } - - if matched { + if combinedFortMatches(&retrieveParameters, &fortLookup, now) { if _, dup := seenCombined[fortId]; dup { return true } diff --git a/decoder/api_fort_combined_test.go b/decoder/api_fort_combined_test.go new file mode 100644 index 00000000..a9228195 --- /dev/null +++ b/decoder/api_fort_combined_test.go @@ -0,0 +1,50 @@ +package decoder + +import "testing" + +// TestCombinedFortMatches locks the typed-group dispatch of the combined scan: +// a clause group only governs its own fort type, so a gym clause can never +// vacuously match a pokestop (its pokestop-facing fields are all wildcards). +func TestCombinedFortMatches(t *testing.T) { + now := int64(1000) + gym := FortLookup{FortType: GYM, RaidLevel: 5, RaidBattleTimestamp: 900, RaidEndTimestamp: 2000} + stop := FortLookup{FortType: POKESTOP, QuestNoArRewardType: 2, QuestNoArRewardItemId: 3} + station := FortLookup{FortType: STATION, StationEndTimestamp: 2000} + + gymClauses := &ApiFortTypeScanGroup{DnfFilters: []ApiFortDnfFilter{{RaidLevel: []int8{5}}}} + + // vacuous cross-type match is impossible: only the gyms group exists + p := &ApiFortCombinedScan{Gyms: gymClauses} + if !combinedFortMatches(p, &gym, now) { + t.Error("tier-5 raid gym should match its own group") + } + if combinedFortMatches(p, &stop, now) { + t.Error("pokestop must NOT match when its group is omitted (excluded type)") + } + if combinedFortMatches(p, &station, now) { + t.Error("station must NOT match when its group is omitted") + } + + // group present with no clauses = match-all for that type only + p = &ApiFortCombinedScan{Gyms: gymClauses, Pokestops: &ApiFortTypeScanGroup{}} + if !combinedFortMatches(p, &stop, now) { + t.Error("empty pokestop group should match every pokestop") + } + if combinedFortMatches(p, &station, now) { + t.Error("station still excluded") + } + + // all groups omitted = bare probe, everything matches + p = &ApiFortCombinedScan{} + for _, fl := range []*FortLookup{&gym, &stop, &station} { + if !combinedFortMatches(p, fl, now) { + t.Errorf("bare probe should match fort type %v", fl.FortType) + } + } + + // clauses within a group still narrow + p = &ApiFortCombinedScan{Pokestops: &ApiFortTypeScanGroup{DnfFilters: []ApiFortDnfFilter{{QuestRewardType: []int16{7}}}}} + if combinedFortMatches(p, &stop, now) { + t.Error("item-quest stop must not match an encounter-only clause") + } +} diff --git a/routes_huma.go b/routes_huma.go index 65d892b8..ffc69203 100644 --- a/routes_huma.go +++ b/routes_huma.go @@ -132,7 +132,7 @@ type pokestopScanOutput struct{ Body decoder.ApiPokestopScanResult } type stationScanInput struct{ Body decoder.ApiFortScan } type stationScanOutput struct{ Body decoder.ApiStationScanResult } -type fortScanInput struct{ Body decoder.ApiFortScan } +type fortScanInput struct{ Body decoder.ApiFortCombinedScan } type fortScanOutput struct { Body decoder.ApiFortCombinedScanResult } @@ -213,7 +213,7 @@ func registerFortScanRoutes(api huma.API) { Method: http.MethodPost, Path: "/api/fort/scan", Summary: "Search all fort types in a bounding box (DNF filters)", - Description: "Returns gyms, pokestops, and stations within [min,max] matching any DNF filter clause, in a single rtree traversal.", + Description: "Returns the requested fort types within [min,max] in a single rtree traversal. Each present group (gyms/pokestops/stations) opts its type in with its own DNF clause set; omitted groups are excluded. Omitting all three groups returns everything.", Tags: []string{"Fort"}, Security: []map[string][]string{{securitySchemeName: {}}}, DefaultStatus: http.StatusOK, From be9ea3b45004f2faab04c1f3b43acd7de0280b1a Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 16:38:10 +0100 Subject: [PATCH 07/29] docs(spec): combined-scan typed groups + part-2 coalescer plan --- .../2026-07-16-fort-dnf-filtering-design.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md index 711be1d2..b1070871 100644 --- a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md +++ b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md @@ -267,3 +267,19 @@ clauses mirroring `finalTeams`/`finalSlots` (t/g keys) for ALL four gym-layer en (all-gyms/ex/in-battle/ar) — the previous poison and the standalone `is_ar_scan_eligible` gym clause are gone; ex/ar/in-battle remain residual halves of an ANDed condition, so the clauses stay a tight superset. Badge viewing still poisons. + +## 12. Combined-scan typed groups (Part 1 of call-combining, 2026-07-17) + +Discovery: the combined `/api/fort/scan`'s flat `filters` list was semantically unusable for +mixed-type clause sets — `isFortDnfMatch` only evaluates a clause's own-type fields, so a gym clause's +pokestop-facing fields are all wildcards and it vacuously matched every pokestop/station. The body is +now **typed groups** (`gyms`/`pokestops`/`stations`, each `{filters: [...]}`): present group = type +included with its own clauses (empty = type match-all), omitted group = type excluded (subsumes the +deferred `fort_types` scope), all omitted = bare probe. Locked by `TestCombinedFortMatches`. + +**Part 2 (ReactMap consumer, pending measurement):** best-effort coalescing — a ~10-15ms window keyed +(user/session, source, bbox) merges the per-type mem-branch scans (which the client fires in the same +pan tick) into one combined call with each registrant's clauses as its group; late arrivals and any +combined failure fall back to the existing per-type scan. Decide after grepping +`GetFortsInArea - scan time` from production logs: if per-scan times are ~1-3ms the saving is +Golbat-CPU-at-scale only. From d6548b6dd195d4c813428f4620dc66b64e541686 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 16:40:03 +0100 Subject: [PATCH 08/29] docs(spec): part-2 coalescer rejected on measurement (scans are microseconds) --- .../specs/2026-07-16-fort-dnf-filtering-design.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md index b1070871..e13b7892 100644 --- a/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md +++ b/docs/superpowers/specs/2026-07-16-fort-dnf-filtering-design.md @@ -280,6 +280,7 @@ deferred `fort_types` scope), all omitted = bare probe. Locked by `TestCombinedF **Part 2 (ReactMap consumer, pending measurement):** best-effort coalescing — a ~10-15ms window keyed (user/session, source, bbox) merges the per-type mem-branch scans (which the client fires in the same pan tick) into one combined call with each registrant's clauses as its group; late arrivals and any -combined failure fall back to the existing per-type scan. Decide after grepping -`GetFortsInArea - scan time` from production logs: if per-scan times are ~1-3ms the saving is -Golbat-CPU-at-scale only. +combined failure fall back to the existing per-type scan. **Measured 2026-07-17 and REJECTED:** production scan times are 10-175µs per scan (tree size +16k, viewport walks of 2-69 forts) — three scans per pan cost ~0.5ms of Golbat CPU total, and a +coalescing window would add 20-100x more latency than it saves. Per-type scans stay. The typed +groups remain valuable as an API-correctness fix for any future consumer of the combined endpoint. From dbcd98658d4592c92a8f4c4ce09aa6f784b76f04 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 18:02:02 +0100 Subject: [PATCH 09/29] =?UTF-8?q?feat(api):=20combined=20GET=20/api/fort/a?= =?UTF-8?q?vailable=20=E2=80=94=20one=20pass=20for=20all=20fort=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three per-type availability endpoints each range the ENTIRE fort cache skipping foreign types; on large instances (~900k forts observed) each walk costs ~100ms and consumers refresh all of them together (often per user session-init). The per-type builders' accumulation is extracted into shared accumulators (gymAvailAcc/pokestopAvailAcc/stationAvailAcc — per-type endpoints unchanged in behavior), and GET /api/fort/available produces all three sections from a single range: {pokestops, gyms, stations}, same inner shapes as the per-type endpoints. Locked by a parity test. Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available.go | 46 +++++++++++ decoder/api_fort_available_test.go | 51 ++++++++++++ decoder/api_gym_available.go | 56 +++++++++----- decoder/api_pokestop_available.go | 120 +++++++++++++++++------------ decoder/api_station_available.go | 68 +++++++++------- routes_huma.go | 21 +++++ 6 files changed, 264 insertions(+), 98 deletions(-) create mode 100644 decoder/api_fort_available.go create mode 100644 decoder/api_fort_available_test.go diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go new file mode 100644 index 00000000..ab9b6a1d --- /dev/null +++ b/decoder/api_fort_available.go @@ -0,0 +1,46 @@ +package decoder + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +// ApiAvailableForts is the whole-instance availability snapshot for every fort +// type, served by GET /api/fort/available. One fortLookupCache range produces +// all three sections — on large instances this replaces three full-cache +// walks (one per per-type endpoint) with one. +type ApiAvailableForts struct { + Pokestops *ApiAvailablePokestops `json:"pokestops" doc:"Pokestop availability (same shape as /api/pokestop/available)"` + Gyms *ApiAvailableGyms `json:"gyms" doc:"Gym availability (same shape as /api/gym/available)"` + Stations *ApiAvailableStations `json:"stations" doc:"Station availability (same shape as /api/station/available)"` +} + +// GetAvailableForts builds all three availability aggregates in a single +// fortLookupCache range, dispatching each fort to its type's accumulator. +func GetAvailableForts(now int64) *ApiAvailableForts { + start := time.Now() + g, p, s := newGymAvailAcc(), newPokestopAvailAcc(), newStationAvailAcc() + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + switch fl.FortType { + case GYM: + g.ingest(&fl, now) + case POKESTOP: + p.ingest(&fl, now) + case STATION: + s.ingest(&fl, now) + } + return true + }) + res := &ApiAvailableForts{ + Pokestops: p.result(start), + Gyms: g.result(start), + Stations: s.result(start), + } + if statsCollector != nil { + statsCollector.ObserveApiScan("available-forts", time.Since(start).Seconds()) + } + log.Infof("available-forts built in %s: one pass over %d gyms / %d pokestops / %d stations", + time.Since(start), g.forts, p.forts, s.forts) + return res +} diff --git a/decoder/api_fort_available_test.go b/decoder/api_fort_available_test.go new file mode 100644 index 00000000..8ee717e9 --- /dev/null +++ b/decoder/api_fort_available_test.go @@ -0,0 +1,51 @@ +package decoder + +import ( + "testing" + + "github.com/puzpuzpuz/xsync/v4" +) + +// TestGetAvailableForts locks that the single-pass combined builder produces +// the same aggregates as the three per-type builders over the same cache. +func TestGetAvailableForts(t *testing.T) { + fortLookupCache = xsync.NewMap[string, FortLookup]() + initQuestConditions() + now := int64(1_000_000) + + fortLookupCache.Store("g1", FortLookup{ + FortType: GYM, TeamId: 1, AvailableSlots: 2, + RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: now + 100, + }) + fortLookupCache.Store("p1", FortLookup{ + FortType: POKESTOP, LureId: 501, LureExpireTimestamp: now + 100, + Incidents: []FortLookupIncident{{Character: 5, DisplayType: 1, ExpireTimestamp: now + 100}}, + }) + fortLookupCache.Store("s1", FortLookup{ + FortType: STATION, + StationBattles: []FortLookupStationBattle{ + {BattleLevel: 5, BattlePokemonId: 150, BattleEndTimestamp: now + 100}, + }, + }) + + combined := GetAvailableForts(now) + if len(combined.Gyms.Teams) != 1 || len(combined.Gyms.Raids) != 1 { + t.Fatalf("gyms: %+v", combined.Gyms) + } + if len(combined.Pokestops.Lures) != 1 || len(combined.Pokestops.Invasions) != 1 { + t.Fatalf("pokestops: %+v", combined.Pokestops) + } + if len(combined.Stations.Battles) != 1 { + t.Fatalf("stations: %+v", combined.Stations) + } + + // parity with the per-type builders over the same cache + perGym := GetAvailableGyms(now) + perStop := GetAvailablePokestops(now) + perStation := GetAvailableStations(now) + if len(perGym.Teams) != len(combined.Gyms.Teams) || + len(perStop.Lures) != len(combined.Pokestops.Lures) || + len(perStation.Battles) != len(combined.Stations.Battles) { + t.Fatal("combined diverges from per-type builders") + } +} diff --git a/decoder/api_gym_available.go b/decoder/api_gym_available.go index 21a7dc21..621af71f 100644 --- a/decoder/api_gym_available.go +++ b/decoder/api_gym_available.go @@ -34,38 +34,52 @@ type ApiAvailableGyms struct { // range over resident gyms — no maintained map (FortLookup carries every gym // filter field). Teams are all-resident (no time filter); raids require an // unexpired raid with level > 0. -func GetAvailableGyms(now int64) *ApiAvailableGyms { - start := time.Now() - res := &ApiAvailableGyms{Teams: []ApiGymTeamAvailable{}, Raids: []ApiGymRaidAvailable{}} - teams := map[ApiGymTeamAvailable]int{} - raids := map[ApiGymRaidAvailable]int{} - forts := 0 +// gymAvailAcc accumulates the gym availability aggregate; ingest assumes the +// fort is a GYM. Shared by the per-type and combined builders. +type gymAvailAcc struct { + teams map[ApiGymTeamAvailable]int + raids map[ApiGymRaidAvailable]int + forts int +} - fortLookupCache.Range(func(_ string, fl FortLookup) bool { - if fl.FortType != GYM { - return true - } - forts++ - teams[ApiGymTeamAvailable{TeamId: fl.TeamId, AvailableSlots: fl.AvailableSlots}]++ - if fl.RaidLevel > 0 && fl.RaidEndTimestamp > now { - raids[ApiGymRaidAvailable{RaidLevel: fl.RaidLevel, PokemonId: fl.RaidPokemonId, Form: fl.RaidPokemonForm}]++ - } - return true - }) +func newGymAvailAcc() *gymAvailAcc { + return &gymAvailAcc{teams: map[ApiGymTeamAvailable]int{}, raids: map[ApiGymRaidAvailable]int{}} +} - for k, n := range teams { +func (a *gymAvailAcc) ingest(fl *FortLookup, now int64) { + a.forts++ + a.teams[ApiGymTeamAvailable{TeamId: fl.TeamId, AvailableSlots: fl.AvailableSlots}]++ + if fl.RaidLevel > 0 && fl.RaidEndTimestamp > now { + a.raids[ApiGymRaidAvailable{RaidLevel: fl.RaidLevel, PokemonId: fl.RaidPokemonId, Form: fl.RaidPokemonForm}]++ + } +} + +func (a *gymAvailAcc) result(start time.Time) *ApiAvailableGyms { + res := &ApiAvailableGyms{Teams: []ApiGymTeamAvailable{}, Raids: []ApiGymRaidAvailable{}} + for k, n := range a.teams { k.Count = n res.Teams = append(res.Teams, k) } - for k, n := range raids { + for k, n := range a.raids { k.Count = n res.Raids = append(res.Raids, k) } - if statsCollector != nil { statsCollector.ObserveApiScan("available-gyms", time.Since(start).Seconds()) } log.Infof("available-gyms built in %s: scanned %d gyms -> %d team/slot, %d raid options", - time.Since(start), forts, len(res.Teams), len(res.Raids)) + time.Since(start), a.forts, len(res.Teams), len(res.Raids)) return res } + +func GetAvailableGyms(now int64) *ApiAvailableGyms { + start := time.Now() + acc := newGymAvailAcc() + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + if fl.FortType == GYM { + acc.ingest(&fl, now) + } + return true + }) + return acc.result(start) +} diff --git a/decoder/api_pokestop_available.go b/decoder/api_pokestop_available.go index 640e8d5a..6c44de7d 100644 --- a/decoder/api_pokestop_available.go +++ b/decoder/api_pokestop_available.go @@ -67,8 +67,56 @@ type ApiAvailablePokestops struct { // tallies FortLookup's own quest-reward fields and cross-checks that tally // against the maintained map (verifyQuestAggregate) to catch reconciliation // drift between the two. -func GetAvailablePokestops(now int64) *ApiAvailablePokestops { - start := time.Now() +// pokestopAvailAcc accumulates the pokestop availability aggregate; ingest +// assumes the fort is a POKESTOP. Shared by the per-type and combined builders. +type pokestopAvailAcc struct { + lures map[int16]int + shows map[ApiPokestopShowcaseAvailable]int // key without Count + inv map[ApiPokestopInvasionAvailable]int // key without Count + rewards map[questRewardKey]int // FortLookup reward tally — cross-checks the maintained map + forts, incidents int +} + +func newPokestopAvailAcc() *pokestopAvailAcc { + return &pokestopAvailAcc{ + lures: map[int16]int{}, + shows: map[ApiPokestopShowcaseAvailable]int{}, + inv: map[ApiPokestopInvasionAvailable]int{}, + rewards: map[questRewardKey]int{}, + } +} + +func (a *pokestopAvailAcc) ingest(fl *FortLookup, now int64) { + a.forts++ + if fl.LureId != 0 && fl.LureExpireTimestamp > now { + a.lures[fl.LureId]++ + } + if fl.ContestPokemonId != 0 && fl.ShowcaseExpiry > now { + a.shows[ApiPokestopShowcaseAvailable{PokemonId: fl.ContestPokemonId, Form: fl.ContestPokemonForm, TypeId: fl.ContestPokemonType}]++ + } + for _, in := range fl.Incidents { + if in.ExpireTimestamp <= now { + continue + } + a.incidents++ + a.inv[ApiPokestopInvasionAvailable{ + Character: in.Character, DisplayType: int16(in.DisplayType), Confirmed: in.Confirmed, + Slot1PokemonId: in.Slot1PokemonId, Slot1Form: in.Slot1Form, + }]++ + } + // FortLookup QuestNoAr* mirrors quest_* (the AR quest → with_ar=true); + // QuestAr* mirrors alternative_quest_* (non-AR → with_ar=false). Must + // match the questConditionKeysFromPokestop convention or the cross-check + // cries wolf. + if fl.QuestNoArRewardType != 0 { + a.rewards[questRewardKey{true, fl.QuestNoArRewardType, fl.QuestNoArRewardItemId, fl.QuestNoArRewardAmount, fl.QuestNoArRewardPokemonId, fl.QuestNoArRewardPokemonForm}]++ + } + if fl.QuestArRewardType != 0 { + a.rewards[questRewardKey{false, fl.QuestArRewardType, fl.QuestArRewardItemId, fl.QuestArRewardAmount, fl.QuestArRewardPokemonId, fl.QuestArRewardPokemonForm}]++ + } +} + +func (a *pokestopAvailAcc) result(start time.Time) *ApiAvailablePokestops { // Initialize the slices so empty categories marshal as [] rather than null. res := &ApiAvailablePokestops{ Quests: []ApiPokestopQuestAvailable{}, @@ -76,71 +124,43 @@ func GetAvailablePokestops(now int64) *ApiAvailablePokestops { Lures: []ApiPokestopLureAvailable{}, Showcases: []ApiPokestopShowcaseAvailable{}, } - forts, incidents := 0, 0 - lures := map[int16]int{} - shows := map[ApiPokestopShowcaseAvailable]int{} // key without Count - inv := map[ApiPokestopInvasionAvailable]int{} // key without Count - // Quests (rewards + title/target) come solely from the maintained conditions map — distinct+counted. // ApiQuestConditionResult and ApiPokestopQuestAvailable share identical fields (name/order/type), so // a direct conversion carries every field without restating them. for _, c := range GetAvailableQuestConditions() { res.Quests = append(res.Quests, ApiPokestopQuestAvailable(c)) } - - // ONE range: lures + showcases + invasions (response) + a quest-reward tally (verification only). - rewards := map[questRewardKey]int{} // direct FortLookup reward count — cross-checks the maintained map - fortLookupCache.Range(func(_ string, fl FortLookup) bool { - if fl.FortType != POKESTOP { - return true - } - forts++ - if fl.LureId != 0 && fl.LureExpireTimestamp > now { - lures[fl.LureId]++ - } - if fl.ContestPokemonId != 0 && fl.ShowcaseExpiry > now { - shows[ApiPokestopShowcaseAvailable{PokemonId: fl.ContestPokemonId, Form: fl.ContestPokemonForm, TypeId: fl.ContestPokemonType}]++ - } - for _, in := range fl.Incidents { - if in.ExpireTimestamp <= now { - continue - } - incidents++ - inv[ApiPokestopInvasionAvailable{ - Character: in.Character, DisplayType: int16(in.DisplayType), Confirmed: in.Confirmed, - Slot1PokemonId: in.Slot1PokemonId, Slot1Form: in.Slot1Form, - }]++ - } - // FortLookup QuestNoAr* mirrors quest_* (the AR quest → with_ar=true); - // QuestAr* mirrors alternative_quest_* (non-AR → with_ar=false). Must - // match the questConditionKeysFromPokestop convention or the cross-check - // cries wolf. - if fl.QuestNoArRewardType != 0 { - rewards[questRewardKey{true, fl.QuestNoArRewardType, fl.QuestNoArRewardItemId, fl.QuestNoArRewardAmount, fl.QuestNoArRewardPokemonId, fl.QuestNoArRewardPokemonForm}]++ - } - if fl.QuestArRewardType != 0 { - rewards[questRewardKey{false, fl.QuestArRewardType, fl.QuestArRewardItemId, fl.QuestArRewardAmount, fl.QuestArRewardPokemonId, fl.QuestArRewardPokemonForm}]++ - } - return true - }) - - for id, n := range lures { + for id, n := range a.lures { res.Lures = append(res.Lures, ApiPokestopLureAvailable{LureId: id, Count: n}) } - for k, n := range shows { + for k, n := range a.shows { k.Count = n res.Showcases = append(res.Showcases, k) } - for k, n := range inv { + for k, n := range a.inv { k.Count = n res.Invasions = append(res.Invasions, k) } - - verifyQuestAggregate(rewards) // alert if the maintained map drifted from the direct FortLookup tally - logAvailablePokestops(time.Since(start), forts, incidents, res) + verifyQuestAggregate(a.rewards) // alert if the maintained map drifted from the direct FortLookup tally + logAvailablePokestops(time.Since(start), a.forts, a.incidents, res) return res } +// GetAvailablePokestops: one range over resident pokestops (lures, showcases, +// invasions + the quest-reward verification tally); quest options come from +// the maintained conditions map in result(). +func GetAvailablePokestops(now int64) *ApiAvailablePokestops { + start := time.Now() + acc := newPokestopAvailAcc() + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + if fl.FortType == POKESTOP { + acc.ingest(&fl, now) + } + return true + }) + return acc.result(start) +} + // questRewardKey is the reward signature shared by the maintained conditions map (minus title/target) // and the FortLookup reward tally used to detect reconciliation drift. type questRewardKey struct { diff --git a/decoder/api_station_available.go b/decoder/api_station_available.go index 802212b0..dc66502c 100644 --- a/decoder/api_station_available.go +++ b/decoder/api_station_available.go @@ -26,43 +26,57 @@ type ApiAvailableStations struct { // StationBattles slice when present, else fall back to the top-battle // projection; skip expired and level-0 battles. // Unlike isFortDnfMatch, level-0 battles are excluded here (ReactMap's !battle_level convention). -func GetAvailableStations(now int64) *ApiAvailableStations { - start := time.Now() - res := &ApiAvailableStations{Battles: []ApiStationBattleAvailable{}} - battles := map[ApiStationBattleAvailable]int{} - forts := 0 +// stationAvailAcc accumulates the station availability aggregate; ingest +// assumes the fort is a STATION. Shared by the per-type and combined builders. +type stationAvailAcc struct { + battles map[ApiStationBattleAvailable]int + forts int +} - add := func(level int8, pokemonId, form int16, end int64) { - if level == 0 || end <= now { - return - } - battles[ApiStationBattleAvailable{BattleLevel: level, PokemonId: pokemonId, Form: form}]++ +func newStationAvailAcc() *stationAvailAcc { + return &stationAvailAcc{battles: map[ApiStationBattleAvailable]int{}} +} + +func (a *stationAvailAcc) add(level int8, pokemonId, form int16, end, now int64) { + if level == 0 || end <= now { + return } + a.battles[ApiStationBattleAvailable{BattleLevel: level, PokemonId: pokemonId, Form: form}]++ +} - fortLookupCache.Range(func(_ string, fl FortLookup) bool { - if fl.FortType != STATION { - return true - } - forts++ - if len(fl.StationBattles) == 0 { - add(fl.BattleLevel, fl.BattlePokemonId, fl.BattlePokemonForm, fl.BattleEndTimestamp) - return true - } - for _, b := range fl.StationBattles { - add(b.BattleLevel, b.BattlePokemonId, b.BattlePokemonForm, b.BattleEndTimestamp) - } - return true - }) +func (a *stationAvailAcc) ingest(fl *FortLookup, now int64) { + a.forts++ + if len(fl.StationBattles) == 0 { + a.add(fl.BattleLevel, fl.BattlePokemonId, fl.BattlePokemonForm, fl.BattleEndTimestamp, now) + return + } + for _, b := range fl.StationBattles { + a.add(b.BattleLevel, b.BattlePokemonId, b.BattlePokemonForm, b.BattleEndTimestamp, now) + } +} - for k, n := range battles { +func (a *stationAvailAcc) result(start time.Time) *ApiAvailableStations { + res := &ApiAvailableStations{Battles: []ApiStationBattleAvailable{}} + for k, n := range a.battles { k.Count = n res.Battles = append(res.Battles, k) } - if statsCollector != nil { statsCollector.ObserveApiScan("available-stations", time.Since(start).Seconds()) } log.Infof("available-stations built in %s: scanned %d stations -> %d battle options", - time.Since(start), forts, len(res.Battles)) + time.Since(start), a.forts, len(res.Battles)) return res } + +func GetAvailableStations(now int64) *ApiAvailableStations { + start := time.Now() + acc := newStationAvailAcc() + fortLookupCache.Range(func(_ string, fl FortLookup) bool { + if fl.FortType == STATION { + acc.ingest(&fl, now) + } + return true + }) + return acc.result(start) +} diff --git a/routes_huma.go b/routes_huma.go index ffc69203..321f742c 100644 --- a/routes_huma.go +++ b/routes_huma.go @@ -145,6 +145,9 @@ type gymAvailableOutput struct { Body *decoder.ApiAvailableGyms } +type fortAvailableOutput struct { + Body *decoder.ApiAvailableForts +} type stationAvailableOutput struct { Body *decoder.ApiAvailableStations } @@ -262,6 +265,24 @@ func registerFortScanRoutes(api huma.API) { return &gymAvailableOutput{Body: decoder.GetAvailableGyms(time.Now().Unix())}, nil }) + fortAvailableOp := huma.Operation{ + OperationID: "available-forts", + Method: http.MethodGet, + Path: "/api/fort/available", + Summary: "List available options for all fort types in one pass", + Description: "Pokestop, gym, and station availability aggregates (same shapes as the per-type /available endpoints) built from a single in-memory cache pass — use this instead of three per-type calls when refreshing everything. Whole-instance; requires fort_in_memory (503 otherwise).", + Tags: []string{"Fort"}, + Security: []map[string][]string{{securitySchemeName: {}}}, + DefaultStatus: http.StatusOK, + } + draftBadge(&fortAvailableOp) + huma.Register(api, fortAvailableOp, func(ctx context.Context, _ *struct{}) (*fortAvailableOutput, error) { + if !config.Config.FortInMemory { + return nil, huma.Error503ServiceUnavailable("fort_in_memory not enabled") + } + return &fortAvailableOutput{Body: decoder.GetAvailableForts(time.Now().Unix())}, nil + }) + stationAvailableOp := huma.Operation{ OperationID: "available-stations", Method: http.MethodGet, From 09cd5b93411fc0ac8e4a5ff901e758c66b7703fb Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 19:37:51 +0100 Subject: [PATCH 10/29] fix(api): combined available logs one line, not one per type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-type accumulator result() finalizers logged "available- built", and GetAvailableForts calls all three — so one combined /api/fort/available call (one cache Range) emitted four log lines (pokestops, gyms, stations, then forts), reading as if it ran four scans. Move logging and the pokestop quest cross-check out of result() into the per-type Get* callers; the combined builder now emits a single "available-forts built" line. Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available.go | 9 ++++++--- decoder/api_gym_available.go | 17 ++++++++++------- decoder/api_pokestop_available.go | 11 +++++++---- decoder/api_station_available.go | 16 +++++++++------- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go index ab9b6a1d..11abfa46 100644 --- a/decoder/api_fort_available.go +++ b/decoder/api_fort_available.go @@ -32,11 +32,14 @@ func GetAvailableForts(now int64) *ApiAvailableForts { } return true }) + // result() is pure — the combined builder emits ONE log line, not one per + // per-type finalizer. res := &ApiAvailableForts{ - Pokestops: p.result(start), - Gyms: g.result(start), - Stations: s.result(start), + Pokestops: p.result(), + Gyms: g.result(), + Stations: s.result(), } + verifyQuestAggregate(p.rewards) // same pokestop cross-check the per-type build runs if statsCollector != nil { statsCollector.ObserveApiScan("available-forts", time.Since(start).Seconds()) } diff --git a/decoder/api_gym_available.go b/decoder/api_gym_available.go index 621af71f..29269ba5 100644 --- a/decoder/api_gym_available.go +++ b/decoder/api_gym_available.go @@ -54,7 +54,9 @@ func (a *gymAvailAcc) ingest(fl *FortLookup, now int64) { } } -func (a *gymAvailAcc) result(start time.Time) *ApiAvailableGyms { +// result is a pure finalizer — no logging (the caller owns the log line so the +// combined builder doesn't emit a spurious per-type "built" entry). +func (a *gymAvailAcc) result() *ApiAvailableGyms { res := &ApiAvailableGyms{Teams: []ApiGymTeamAvailable{}, Raids: []ApiGymRaidAvailable{}} for k, n := range a.teams { k.Count = n @@ -64,11 +66,6 @@ func (a *gymAvailAcc) result(start time.Time) *ApiAvailableGyms { k.Count = n res.Raids = append(res.Raids, k) } - if statsCollector != nil { - statsCollector.ObserveApiScan("available-gyms", time.Since(start).Seconds()) - } - log.Infof("available-gyms built in %s: scanned %d gyms -> %d team/slot, %d raid options", - time.Since(start), a.forts, len(res.Teams), len(res.Raids)) return res } @@ -81,5 +78,11 @@ func GetAvailableGyms(now int64) *ApiAvailableGyms { } return true }) - return acc.result(start) + res := acc.result() + if statsCollector != nil { + statsCollector.ObserveApiScan("available-gyms", time.Since(start).Seconds()) + } + log.Infof("available-gyms built in %s: scanned %d gyms -> %d team/slot, %d raid options", + time.Since(start), acc.forts, len(res.Teams), len(res.Raids)) + return res } diff --git a/decoder/api_pokestop_available.go b/decoder/api_pokestop_available.go index 6c44de7d..c5ae835b 100644 --- a/decoder/api_pokestop_available.go +++ b/decoder/api_pokestop_available.go @@ -116,7 +116,9 @@ func (a *pokestopAvailAcc) ingest(fl *FortLookup, now int64) { } } -func (a *pokestopAvailAcc) result(start time.Time) *ApiAvailablePokestops { +// result is a pure finalizer — no logging or verification (the caller owns +// those, so the combined builder doesn't emit a spurious per-type entry). +func (a *pokestopAvailAcc) result() *ApiAvailablePokestops { // Initialize the slices so empty categories marshal as [] rather than null. res := &ApiAvailablePokestops{ Quests: []ApiPokestopQuestAvailable{}, @@ -141,8 +143,6 @@ func (a *pokestopAvailAcc) result(start time.Time) *ApiAvailablePokestops { k.Count = n res.Invasions = append(res.Invasions, k) } - verifyQuestAggregate(a.rewards) // alert if the maintained map drifted from the direct FortLookup tally - logAvailablePokestops(time.Since(start), a.forts, a.incidents, res) return res } @@ -158,7 +158,10 @@ func GetAvailablePokestops(now int64) *ApiAvailablePokestops { } return true }) - return acc.result(start) + res := acc.result() + verifyQuestAggregate(acc.rewards) // alert if the maintained map drifted from the direct FortLookup tally + logAvailablePokestops(time.Since(start), acc.forts, acc.incidents, res) + return res } // questRewardKey is the reward signature shared by the maintained conditions map (minus title/target) diff --git a/decoder/api_station_available.go b/decoder/api_station_available.go index dc66502c..2be0bb16 100644 --- a/decoder/api_station_available.go +++ b/decoder/api_station_available.go @@ -55,17 +55,13 @@ func (a *stationAvailAcc) ingest(fl *FortLookup, now int64) { } } -func (a *stationAvailAcc) result(start time.Time) *ApiAvailableStations { +// result is a pure finalizer — no logging (see gymAvailAcc.result). +func (a *stationAvailAcc) result() *ApiAvailableStations { res := &ApiAvailableStations{Battles: []ApiStationBattleAvailable{}} for k, n := range a.battles { k.Count = n res.Battles = append(res.Battles, k) } - if statsCollector != nil { - statsCollector.ObserveApiScan("available-stations", time.Since(start).Seconds()) - } - log.Infof("available-stations built in %s: scanned %d stations -> %d battle options", - time.Since(start), a.forts, len(res.Battles)) return res } @@ -78,5 +74,11 @@ func GetAvailableStations(now int64) *ApiAvailableStations { } return true }) - return acc.result(start) + res := acc.result() + if statsCollector != nil { + statsCollector.ObserveApiScan("available-stations", time.Since(start).Seconds()) + } + log.Infof("available-stations built in %s: scanned %d stations -> %d battle options", + time.Since(start), acc.forts, len(res.Battles)) + return res } From 26a7cc1eb9e763b192f744893f7a5ed898f63965 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 20:24:44 +0100 Subject: [PATCH 11/29] refactor(api): drop team/slot from gym availability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every (team, available-slots) combination exists on a live instance, and consumers generate the t/g filter keys statically — so aggregating them per availability build was redundant. GetAvailableGyms (and the combined /api/fort/available gyms section) now returns raids only. FortLookup keeps TeamId/AvailableSlots — the DNF gym-layer narrowing still uses them. Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available_test.go | 4 ++-- decoder/api_gym_available.go | 27 ++++++++------------------- decoder/api_gym_available_test.go | 3 --- routes_huma.go | 4 ++-- 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/decoder/api_fort_available_test.go b/decoder/api_fort_available_test.go index 8ee717e9..d53fac35 100644 --- a/decoder/api_fort_available_test.go +++ b/decoder/api_fort_available_test.go @@ -29,7 +29,7 @@ func TestGetAvailableForts(t *testing.T) { }) combined := GetAvailableForts(now) - if len(combined.Gyms.Teams) != 1 || len(combined.Gyms.Raids) != 1 { + if len(combined.Gyms.Raids) != 1 { t.Fatalf("gyms: %+v", combined.Gyms) } if len(combined.Pokestops.Lures) != 1 || len(combined.Pokestops.Invasions) != 1 { @@ -43,7 +43,7 @@ func TestGetAvailableForts(t *testing.T) { perGym := GetAvailableGyms(now) perStop := GetAvailablePokestops(now) perStation := GetAvailableStations(now) - if len(perGym.Teams) != len(combined.Gyms.Teams) || + if len(perGym.Raids) != len(combined.Gyms.Raids) || len(perStop.Lures) != len(combined.Pokestops.Lures) || len(perStation.Battles) != len(combined.Stations.Battles) { t.Fatal("combined diverges from per-type builders") diff --git a/decoder/api_gym_available.go b/decoder/api_gym_available.go index 29269ba5..a2fb0500 100644 --- a/decoder/api_gym_available.go +++ b/decoder/api_gym_available.go @@ -6,14 +6,6 @@ import ( log "github.com/sirupsen/logrus" ) -// ApiGymTeamAvailable is one distinct (team, available-slots) pair present on -// resident gyms, with how many gyms carry it. ReactMap derives its t/g keys. -type ApiGymTeamAvailable struct { - TeamId int8 `json:"team_id" doc:"Controlling team id (0 = uncontested)"` - AvailableSlots int8 `json:"available_slots" doc:"Open defender slots"` - Count int `json:"count" doc:"Number of resident gyms with this team/slots"` -} - // ApiGymRaidAvailable is one distinct active raid option on resident gyms. // PokemonId 0 means an egg (no boss yet). ReactMap derives its e/r/boss keys. type ApiGymRaidAvailable struct { @@ -25,8 +17,11 @@ type ApiGymRaidAvailable struct { // ApiAvailableGyms is the whole-instance gym filter snapshot served by // GET /api/gym/available. +// ApiAvailableGyms is the whole-instance gym filter snapshot. Only raids are +// dynamic — team/slot filter keys are generated statically by the consumer +// (every team/slot combination exists on a live instance), so they are not +// aggregated here. type ApiAvailableGyms struct { - Teams []ApiGymTeamAvailable `json:"teams" doc:"Distinct team + available-slot pairs on resident gyms"` Raids []ApiGymRaidAvailable `json:"raids" doc:"Distinct active raid levels/bosses/eggs on resident gyms"` } @@ -37,18 +32,16 @@ type ApiAvailableGyms struct { // gymAvailAcc accumulates the gym availability aggregate; ingest assumes the // fort is a GYM. Shared by the per-type and combined builders. type gymAvailAcc struct { - teams map[ApiGymTeamAvailable]int raids map[ApiGymRaidAvailable]int forts int } func newGymAvailAcc() *gymAvailAcc { - return &gymAvailAcc{teams: map[ApiGymTeamAvailable]int{}, raids: map[ApiGymRaidAvailable]int{}} + return &gymAvailAcc{raids: map[ApiGymRaidAvailable]int{}} } func (a *gymAvailAcc) ingest(fl *FortLookup, now int64) { a.forts++ - a.teams[ApiGymTeamAvailable{TeamId: fl.TeamId, AvailableSlots: fl.AvailableSlots}]++ if fl.RaidLevel > 0 && fl.RaidEndTimestamp > now { a.raids[ApiGymRaidAvailable{RaidLevel: fl.RaidLevel, PokemonId: fl.RaidPokemonId, Form: fl.RaidPokemonForm}]++ } @@ -57,11 +50,7 @@ func (a *gymAvailAcc) ingest(fl *FortLookup, now int64) { // result is a pure finalizer — no logging (the caller owns the log line so the // combined builder doesn't emit a spurious per-type "built" entry). func (a *gymAvailAcc) result() *ApiAvailableGyms { - res := &ApiAvailableGyms{Teams: []ApiGymTeamAvailable{}, Raids: []ApiGymRaidAvailable{}} - for k, n := range a.teams { - k.Count = n - res.Teams = append(res.Teams, k) - } + res := &ApiAvailableGyms{Raids: []ApiGymRaidAvailable{}} for k, n := range a.raids { k.Count = n res.Raids = append(res.Raids, k) @@ -82,7 +71,7 @@ func GetAvailableGyms(now int64) *ApiAvailableGyms { if statsCollector != nil { statsCollector.ObserveApiScan("available-gyms", time.Since(start).Seconds()) } - log.Infof("available-gyms built in %s: scanned %d gyms -> %d team/slot, %d raid options", - time.Since(start), acc.forts, len(res.Teams), len(res.Raids)) + log.Infof("available-gyms built in %s: scanned %d gyms -> %d raid options", + time.Since(start), acc.forts, len(res.Raids)) return res } diff --git a/decoder/api_gym_available_test.go b/decoder/api_gym_available_test.go index f3118011..7833abc6 100644 --- a/decoder/api_gym_available_test.go +++ b/decoder/api_gym_available_test.go @@ -29,9 +29,6 @@ func TestGetAvailableGyms(t *testing.T) { res := GetAvailableGyms(now) - if len(res.Teams) != 3 { // (1,2),(2,6),(1,0) - t.Fatalf("teams: %+v", res.Teams) - } // raids: boss 150 lvl5, egg lvl3; expired 999 excluded var bosses, eggs int for _, r := range res.Raids { diff --git a/routes_huma.go b/routes_huma.go index 321f742c..e4bde12c 100644 --- a/routes_huma.go +++ b/routes_huma.go @@ -251,8 +251,8 @@ func registerFortScanRoutes(api huma.API) { OperationID: "available-gyms", Method: http.MethodGet, Path: "/api/gym/available", - Summary: "List currently available gym teams/slots and raid options", - Description: "Distinct (team, available-slots) pairs and active raid levels/bosses/eggs on resident gyms, from the in-memory fort cache (no DB scan). Whole-instance; requires fort_in_memory (503 otherwise).", + Summary: "List currently available gym raid options", + Description: "Distinct active raid levels/bosses/eggs on resident gyms, from the in-memory fort cache (no DB scan). Team/slot filter keys are generated statically by consumers, so they are not returned here. Whole-instance; requires fort_in_memory (503 otherwise).", Tags: []string{"Fort"}, Security: []map[string][]string{{securitySchemeName: {}}}, DefaultStatus: http.StatusOK, From 921e06d006027ca19769703b992197a863e6ad6e Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 20:48:07 +0100 Subject: [PATCH 12/29] docs(spec): maintained fort availability index (max-expiry, no scan) --- ...-17-maintained-fort-availability-design.md | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md diff --git a/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md b/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md new file mode 100644 index 00000000..bed85f86 --- /dev/null +++ b/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md @@ -0,0 +1,178 @@ +# Maintained Fort Availability Index — Design Spec + +- **Date:** 2026-07-17 +- **Status:** Approved design → planning +- **Repo:** Golbat only (`feat/fort-scan-map-data`, worktree `~/GolandProjects/Golbat-wt/pokestop-available-api`, PR #385) +- **Author:** James Berry (with Claude) +- **Extends:** the fort availability endpoints (`/api/{pokestop,gym,station}/available` + combined `/api/fort/available`). + +## 1. Problem + +The availability endpoints answer "which filter options exist on any resident fort right now?" — distinct active lures, showcases, invasions (pokestop), raids (gym), and battles (station). Today each build does a full `fortLookupCache.Range` over **every** fort. On a large instance (~1.7M forts) that is ~600 ms, fired ~once/minute (session-init + the `availableRefreshSeconds` TTL). The output is tiny (a few hundred distinct options), so we scan 1.7M entries to produce ~400 keys. + +Quests already avoid the scan — they are served from a maintained running aggregate (`questConditionCount`, updated incrementally via `reconcileFortQuestConditions`). This spec applies the same "maintained, not scanned" principle to the remaining five aggregates, eliminating the per-request full-fort walk entirely. + +## 2. Why a different mechanism than quests + +Quests use **reconcile** (a per-fort contribution tracker + exact counts + decrement) because a quest can be **retracted** mid-life — cleared by `RemoveQuestsWithinGeofence`, replaced by a next-day scan, or swapped by an event — while its nominal `quest_expiry` (≈ daily) is still hours away. Reconcile handles retraction; a monotonic index cannot. + +The five aggregates here are different: **an entry's disappearance is always signalled by its own expiry timestamp passing.** A lure is gone exactly when `LureExpireTimestamp` passes; a raid at `RaidEndTimestamp`; an incident at its `ExpireTimestamp`; a battle at `BattleEndTimestamp`. There is no geofence-clear or early-removal routine for any of them. That property lets us use a far simpler structure than reconcile: + +> **Max-expiry index:** per distinct option value, store the latest expiry seen. On read, emit the value iff `maxExpiry > now`. + +No per-fort tracker, no decrement, no eviction/clear handling, and self-healing in both directions (see §7). + +## 3. Architecture + +Five package-level maps, one per aggregate, each `*xsync.Map[, int64]` mapping an option value to its **max seen expiry** (unix seconds): + +| Aggregate | Map | Option key | Expiry field | Write hook (fn, lock domain) | +|---|---|---|---|---| +| Pokestop lure | `lureExpiry` | `int16` lure id | `LureExpireTimestamp` | `updatePokestopLookup` (pokestop) | +| Pokestop showcase | `showcaseExpiry` | `{PokemonId int16, Form int16, TypeId int8}` | `ShowcaseExpiry` | `updatePokestopLookup` (pokestop) | +| Pokestop invasion | `invasionExpiry` | `{Character int16, DisplayType int16, Confirmed bool, Slot1PokemonId int16, Slot1Form int16}` | incident `ExpireTimestamp` | `updatePokestopIncidentLookup` (incident) | +| Gym raid | `raidExpiry` | `{RaidLevel int8, PokemonId int16, Form int16}` | `RaidEndTimestamp` | `updateGymLookup` (gym) | +| Station battle | `battleExpiry` | `{BattleLevel int8, PokemonId int16, Form int16}` | `BattleEndTimestamp` | `updateStationLookupWithBattles` (station) | + +The option keys are exactly the existing `Api*Available` structs **minus `Count`** (§8) — same fields consumers already receive. + +**Concurrency is trivial:** each map has a single writer lock-domain (lures/showcases from the pokestop domain, invasions from the incident domain, raids from the gym domain, battles from the station domain — none shared), and every write is one atomic `Compute` that keeps the larger expiry. No cross-map invariant, so no lock-order reasoning. Reads (`Range`) run concurrently with writes as `xsync.Map` allows. + +### 3.1 The observe primitive + +```go +// observeExpiry records value as available until at least expiry. Ignores +// already-expired observations (never inserts a dead key). Keeping the LARGER +// expiry means a still-active fort refreshes the option's lifetime. +func observeExpiry[K comparable](m *xsync.Map[K, int64], key K, expiry, now int64) { + if expiry <= now { + return + } + m.Compute(key, func(old int64, _ bool) (int64, xsync.ComputeOp) { + if old >= expiry { + return old, xsync.CancelOp + } + return expiry, xsync.UpdateOp + }) +} +``` + +### 3.2 Write hooks + +Each hook fires inside the existing update function, using the same `now` the function already computes, e.g. in `updateGymLookup`: + +```go +if fl.RaidLevel > 0 { + observeExpiry(raidExpiry, raidKey{fl.RaidLevel, fl.RaidPokemonId, fl.RaidPokemonForm}, fl.RaidEndTimestamp, now) +} +``` + +Invasions observe **per incident** inside `updatePokestopIncidentLookup` (that function already builds the `FortLookupIncident` with `ExpireTimestamp`). Battles observe **per battle** inside `updateStationLookupWithBattles`. Lures and showcases observe once per pokestop in `updatePokestopLookup`. + +These hooks also fire during preload (`preload.go` calls the same update functions), so the maps warm up at startup — no cold-start gap beyond the preload window that already exists. + +### 3.3 Read side (prune-on-read) + +Each `GetAvailable` ranges its map, emits live keys, and deletes the ones whose expiry has passed (so the map stays bounded by *currently-distinct* options, not all-time): + +```go +func readRaidAvailable(now int64) []ApiGymRaidAvailable { + out := []ApiGymRaidAvailable{} + raidExpiry.Range(func(k raidKey, exp int64) bool { + if exp > now { + out = append(out, ApiGymRaidAvailable{RaidLevel: k.level, PokemonId: k.id, Form: k.form}) + return true + } + // Conditional prune: delete ONLY if still expired. A blind Delete could + // race a concurrent observe that just refreshed this key to a future + // expiry and wrongly drop the live option. Compute re-checks under lock. + raidExpiry.Compute(k, func(cur int64, loaded bool) (int64, xsync.ComputeOp) { + if loaded && cur <= now { + return 0, xsync.DeleteOp + } + return cur, xsync.CancelOp + }) + return true + }) + return out +} +``` + +- `GetAvailableGyms` → `{raids: readRaidAvailable(now)}` +- `GetAvailableStations` → `{battles: readBattleAvailable(now)}` +- `GetAvailablePokestops` → `{quests: GetAvailableQuestConditions(), lures: …, showcases: …, invasions: …}` (quests unchanged) +- `GetAvailableForts` → assembles the three from the same reads — **no `fortLookupCache.Range` anywhere** + +## 4. What is removed + +- The `fortLookupCache.Range` full-fort walk in `GetAvailablePokestops`, `GetAvailableGyms`, `GetAvailableStations`, and `GetAvailableForts`. +- The `gymAvailAcc` / `pokestopAvailAcc` / `stationAvailAcc` accumulators (they existed only to tally the scan). +- The `verifyQuestAggregate` cross-check and its FortLookup quest-reward tally (it compared the reconcile map to the scan; with no scan there is nothing to compare — see §7 caveat). +- The `Count` field on every `Api*Available` struct (§8). + +Quests (`questConditionCount` / `reconcileFortQuestConditions` / `GetAvailableQuestConditions`) are **unchanged**. + +## 5. Data flow + +``` +fort save / incident save / preload + │ (existing update fn, same `now`) + ▼ + observeExpiry(map, optionKey, expiry, now) ── atomic max, per-key + │ + ▼ + Expiry map (option → maxExpiry) + ▲ + │ GetAvailable(now): Range → emit exp>now, Delete exp<=now + ▼ + /api/{type}/available and /api/fort/available (no scan) +``` + +## 6. Concurrency & correctness notes + +- Single-writer-domain per map ⇒ writes never race each other on a key beyond `Compute`'s own atomicity. +- Prune-on-read must delete **conditionally** (`Compute` with a delete-if-still-`<= now` predicate, §3.3), never a blind `Delete`: a blind delete could race a concurrent `observe` that just refreshed the key to a future expiry and wrongly drop a live option. The conditional re-check runs under the key's lock, so a refreshed key survives. +- No eviction hook needed: an evicted fort's option remains valid until its expiry passes (the option genuinely is still active until then), then prunes on the next read. + +## 7. Accepted trade-offs (self-healing only, no sweep) + +Per the design decision, there is **no periodic reconciliation sweep**. The consequences, all bounded/cosmetic: + +1. **Over-report on mid-life replacement.** An egg key lingers after it hatches (until the raid's own `RaidEndTimestamp`); a grunt key after it's swapped (until the old incident's `ExpireTimestamp`). Capped at the entry's own expiry (~30–45 min), self-healing, and the option almost always still exists on another fort. Documented, not fought. +2. **No quest drift net.** `verifyQuestAggregate` is removed. It was Debug-level and its own comment calls divergences "benign, transient"; the reconcile map is otherwise rebuilt across the daily clear+rescan cycle. Acceptable. +3. **Rare stuck under-report.** If the *only* fort holding some option never re-saves, that option could be missing from the list. Extremely rare (would need a unique option on a fort that stops being scanned), cosmetic (a filter option absent), and self-heals on any re-save. No mitigation. +4. **Counts gone** (§8). + +## 8. Counts + +The `count` field on the availability responses is **unused by consumers** — ReactMap's `gymAvailableMapper` / `stationAvailableMapper` / `pokestopAvailableMapper` build a distinct key set and never read `count`. Max-expiry does not produce counts. So the `Count` field is **dropped** from `ApiPokestopLureAvailable`, `ApiPokestopShowcaseAvailable`, `ApiPokestopInvasionAvailable`, `ApiGymRaidAvailable`, and `ApiStationBattleAvailable`. This is a visible response-shape change but harmless (no consumer reads it). Pokemon availability is untouched — its count feeds rarity and stays a maintained count. + +## 9. Testing + +Node-golden equivalent is Go unit tests (`_test.go`), matching the existing availability test style: + +- **`observeExpiry`**: keeps the larger expiry; ignores `expiry <= now`; distinct keys independent. +- **Each read fn**: emits keys with `exp > now`; prunes and omits `exp <= now`; empty map → empty slice (not nil, marshals `[]`). +- **Each write hook**: a synthetic fort/incident/battle save (drive the existing update fn or the hook directly) inserts the expected key with the expected expiry; a level-0/lure-0/expired entry inserts nothing. +- **`GetAvailableForts`**: assembles the three sections from seeded maps; parity with the per-type reads over the same maps. +- **No-scan assertion**: the read path performs no `fortLookupCache.Range` (structural — enforced by the accumulators being gone). + +## 10. Non-goals + +- Quests stay on reconcile (they require retraction; max-expiry cannot express it). +- No periodic reconciliation sweep (self-healing only — §7). +- Pokemon availability unchanged (count is load-bearing for rarity). +- ReactMap unchanged — response shapes are identical except the dropped `count`, which ReactMap never reads. +- Gym team/slot availability (already removed) and `station_active` (already shipped) are out of scope. + +## 11. Task decomposition (for the plan) + +Naturally splits into independent, individually-reviewable tasks: + +1. `observeExpiry` primitive + the five map declarations + init. +2. Pokestop aggregates (lures, showcases, invasions): hooks in `updatePokestopLookup` + `updatePokestopIncidentLookup`; `GetAvailablePokestops` reads maps; drop `pokestopAvailAcc`, the scan, and `verifyQuestAggregate`. +3. Gym raids: hook in `updateGymLookup`; `GetAvailableGyms` reads map; drop `gymAvailAcc`. +4. Station battles: hook in `updateStationLookupWithBattles`; `GetAvailableStations` reads map; drop `stationAvailAcc`. +5. `GetAvailableForts` assembles from the three (no range); drop `Count` fields across the response structs. + +Each task ends green (`go build -tags go_json ./...` + `go test ./decoder/`), with the availability endpoints returning the same key sets they do today (minus counts), served from the maps. From b7820105e4619d2a7387860c37394f8d1b6264cb Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 20:54:32 +0100 Subject: [PATCH 13/29] docs(spec): pin strong Range (not RangeRelaxed) for prune-on-read --- .../specs/2026-07-17-maintained-fort-availability-design.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md b/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md index bed85f86..3bcc20af 100644 --- a/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md +++ b/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md @@ -131,6 +131,7 @@ fort save / incident save / preload ## 6. Concurrency & correctness notes - Single-writer-domain per map ⇒ writes never race each other on a key beyond `Compute`'s own atomicity. +- Use the strong `Map.Range` (each key visited at most once), **not** `RangeRelaxed` (which may visit a key more than once → a duplicate availability entry). `xsync/v4 v4.5.0` documents `Range` as safe to modify during iteration, including deletion — so prune-on-read during `Range` is supported. - Prune-on-read must delete **conditionally** (`Compute` with a delete-if-still-`<= now` predicate, §3.3), never a blind `Delete`: a blind delete could race a concurrent `observe` that just refreshed the key to a future expiry and wrongly drop a live option. The conditional re-check runs under the key's lock, so a refreshed key survives. - No eviction hook needed: an evicted fort's option remains valid until its expiry passes (the option genuinely is still active until then), then prunes on the next read. From 5aafcce2ad023272c8e43550b841b1995b9411aa Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 21:00:26 +0100 Subject: [PATCH 14/29] docs(plan): maintained fort availability index (4 tasks) --- ...2026-07-17-maintained-fort-availability.md | 796 ++++++++++++++++++ 1 file changed, 796 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-17-maintained-fort-availability.md diff --git a/docs/superpowers/plans/2026-07-17-maintained-fort-availability.md b/docs/superpowers/plans/2026-07-17-maintained-fort-availability.md new file mode 100644 index 00000000..46dafd47 --- /dev/null +++ b/docs/superpowers/plans/2026-07-17-maintained-fort-availability.md @@ -0,0 +1,796 @@ +# Maintained Fort Availability Index Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the per-request full-fort `fortLookupCache.Range` in the availability endpoints with five maintained max-expiry maps, updated incrementally on fort save. + +**Architecture:** Each dynamic aggregate (pokestop lure/showcase/invasion, gym raid, station battle) gets an `xsync.Map[optionKey, maxExpiry]`. The fort update functions `observe` each active option (atomic keep-larger); the availability builders `Range` the small map, emit `exp > now`, and prune-on-read the rest. Quests keep their existing `reconcile` aggregate untouched. No full-fort scan remains anywhere in the availability path. + +**Tech Stack:** Go, `github.com/puzpuzpuz/xsync/v4` v4.5.0 (`Map.Compute` / strong `Map.Range`), huma. Tests are Go `_test.go` run with `go test ./decoder/`. + +## Global Constraints + +- **Golbat only.** ReactMap is unaffected (response shapes are identical except the dropped `count`, which no consumer reads). Do not touch any ReactMap file. +- **Quests are out of scope and unchanged.** `questConditionCount`, `reconcileFortQuestConditions`, `GetAvailableQuestConditions`, `questFortKeys` stay exactly as they are. Quest availability continues to come from `GetAvailableQuestConditions()`. +- **Prune-on-read is conditional, never blind.** Deleting an expired key must go through `Compute` with a delete-if-still-`<= now` predicate (`pruneExpired`), so it cannot race an `observe` that just refreshed the key. A blind `m.Delete(k)` is a defect. +- **Use strong `Map.Range`, not `RangeRelaxed`.** `RangeRelaxed` may visit a key twice → a duplicate availability entry. +- **`observe` ignores already-expired observations** (`expiry <= now` → no-op) so dead keys never enter a map. +- **Single writer-domain per map.** `lureExpiry`/`showcaseExpiry` are written only from the pokestop lock domain (`updatePokestopLookup`), `invasionExpiry` only from the incident domain (`updatePokestopIncidentLookup`), `raidExpiry` only from the gym domain (`updateGymLookup`), `battleExpiry` only from the station domain (`updateStationLookupWithBattles`). Do not observe an aggregate from a foreign domain. +- **Build + test gate each task:** `go build -tags go_json ./...` and `go test ./decoder/ -count=1` both green before commit. +- **Empty aggregates marshal as `[]`, not `null`** — initialize result slices to `[]ApiXAvailable{}`. +- Commit subjects: Conventional Commits, imperative, lowercase, ≤100 chars; end the body with `Co-Authored-By: Claude Fable 5 `. + +--- + +## File Structure + +- **Create `decoder/fort_availability.go`** — the maintained index: the five key types, the five maps, `initFortAvailability`, the generic `observeExpiry` + `pruneExpired`, and per-aggregate `observe*` / `read*` functions. One responsibility: the max-expiry index. +- **Create `decoder/fort_availability_test.go`** — unit tests for the primitive, observe, and read functions. +- **Modify `decoder/fortRtree.go`** — call `initFortAvailability()` from `initFortRtree`; add `observe*` calls to `updatePokestopLookup`, `updatePokestopIncidentLookup`, `updateGymLookup`, `updateStationLookupWithBattles`. +- **Modify `decoder/api_gym_available.go`** — `GetAvailableGyms` reads `readRaids`; delete `gymAvailAcc`; drop `ApiGymRaidAvailable.Count`. +- **Modify `decoder/api_station_available.go`** — `GetAvailableStations` reads `readBattles`; delete `stationAvailAcc`; drop `ApiStationBattleAvailable.Count`. +- **Modify `decoder/api_pokestop_available.go`** — `GetAvailablePokestops` reads `readLures`/`readShowcases`/`readInvasions` + `GetAvailableQuestConditions`; delete `pokestopAvailAcc`, `verifyQuestAggregate`, `questRewardKey`; drop `Count` on lure/showcase/invasion structs. +- **Modify `decoder/api_fort_available.go`** — `GetAvailableForts` assembles from the read functions; no `fortLookupCache.Range`. +- **Modify the availability `_test.go` files** — seed via `observe*` (or the update hooks) instead of `fortLookupCache`. + +--- + +## Task 1: Maintained-index primitive + gym raids (proves the pattern) + +**Files:** +- Create: `decoder/fort_availability.go` +- Create: `decoder/fort_availability_test.go` +- Modify: `decoder/fortRtree.go` (`initFortRtree`, `updateGymLookup`) +- Modify: `decoder/api_gym_available.go` +- Modify: `decoder/api_gym_available_test.go` + +**Interfaces:** +- Produces (used by Tasks 2-4): + - `func observeExpiry[K comparable](m *xsync.Map[K, int64], key K, expiry, now int64)` + - `func pruneExpired[K comparable](m *xsync.Map[K, int64], key K, now int64)` + - `func initFortAvailability()` + - `raidExpiry *xsync.Map[raidKey, int64]`, `type raidKey struct { RaidLevel int8; PokemonId, Form int16 }` + - `func observeRaid(fl *FortLookup, now int64)`, `func readRaids(now int64) []ApiGymRaidAvailable` + +- [ ] **Step 1: Write the failing test** — `decoder/fort_availability_test.go` + +```go +package decoder + +import "testing" + +func TestObserveExpiryAndReadRaids(t *testing.T) { + initFortAvailability() + now := int64(1000) + + // active raid boss + active egg + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidPokemonForm: 0, RaidEndTimestamp: 2000}, now) + observeRaid(&FortLookup{RaidLevel: 3, RaidPokemonId: 0, RaidPokemonForm: 0, RaidEndTimestamp: 2000}, now) + // no raid (level 0) -> ignored + observeRaid(&FortLookup{RaidLevel: 0, RaidEndTimestamp: 2000}, now) + // already-expired -> ignored + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 999, RaidEndTimestamp: 500}, now) + + got := readRaids(now) + if len(got) != 2 { + t.Fatalf("want 2 raid options, got %d: %+v", len(got), got) + } + for _, r := range got { + if r.PokemonId == 999 { + t.Fatal("expired raid must not appear") + } + } + + // keep-larger: re-observe boss 150 with a LATER expiry, then read after the + // first expiry has passed — it must still be present. + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: 3000}, now) + if len(readRaids(2500)) == 0 { + t.Fatal("refreshed raid should survive past its first expiry") + } + + // prune-on-read: once fully expired, it drops out. + if len(readRaids(4000)) != 0 { + t.Fatal("all raids expired -> empty") + } + // and empty read returns [] not nil + if readRaids(4000) == nil { + t.Fatal("read must return non-nil empty slice") + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./decoder/ -run TestObserveExpiryAndReadRaids -count=1` +Expected: FAIL — `undefined: initFortAvailability` / `observeRaid` / `readRaids`. + +- [ ] **Step 3: Create `decoder/fort_availability.go`** + +```go +package decoder + +import "github.com/puzpuzpuz/xsync/v4" + +// Maintained max-expiry availability index. Each map holds, per distinct filter +// option, the latest expiry timestamp seen on any resident fort. Availability +// reads the maps instead of scanning fortLookupCache; see +// docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md. +// +// Quests are NOT here — they can be retracted mid-life (geofence clear, event +// swap) which a monotonic max-expiry cannot express, so they keep the reconcile +// aggregate (questConditionCount). + +type showcaseKey struct { + PokemonId int16 + Form int16 + TypeId int8 +} + +type invasionKey struct { + Character int16 + DisplayType int16 + Confirmed bool + Slot1PokemonId int16 + Slot1Form int16 +} + +type raidKey struct { + RaidLevel int8 + PokemonId int16 + Form int16 +} + +type battleKey struct { + BattleLevel int8 + PokemonId int16 + Form int16 +} + +var ( + lureExpiry *xsync.Map[int16, int64] + showcaseExpiry *xsync.Map[showcaseKey, int64] + invasionExpiry *xsync.Map[invasionKey, int64] + raidExpiry *xsync.Map[raidKey, int64] + battleExpiry *xsync.Map[battleKey, int64] +) + +func initFortAvailability() { + lureExpiry = xsync.NewMap[int16, int64]() + showcaseExpiry = xsync.NewMap[showcaseKey, int64]() + invasionExpiry = xsync.NewMap[invasionKey, int64]() + raidExpiry = xsync.NewMap[raidKey, int64]() + battleExpiry = xsync.NewMap[battleKey, int64]() +} + +// observeExpiry records key as available until at least expiry, keeping the +// larger of any prior expiry (a still-active fort refreshes the lifetime). +// Already-expired observations are ignored so dead keys never enter the map. +func observeExpiry[K comparable](m *xsync.Map[K, int64], key K, expiry, now int64) { + if expiry <= now { + return + } + m.Compute(key, func(old int64, _ bool) (int64, xsync.ComputeOp) { + if old >= expiry { + return old, xsync.CancelOp + } + return expiry, xsync.UpdateOp + }) +} + +// pruneExpired deletes key iff it is STILL expired. It must never be a blind +// Delete: that could race an observe that just refreshed the key and wrongly +// drop a live option. Compute re-checks under the key's lock. +func pruneExpired[K comparable](m *xsync.Map[K, int64], key K, now int64) { + m.Compute(key, func(cur int64, loaded bool) (int64, xsync.ComputeOp) { + if loaded && cur <= now { + return 0, xsync.DeleteOp + } + return cur, xsync.CancelOp + }) +} + +func observeRaid(fl *FortLookup, now int64) { + if fl.RaidLevel > 0 { + observeExpiry(raidExpiry, raidKey{fl.RaidLevel, fl.RaidPokemonId, fl.RaidPokemonForm}, fl.RaidEndTimestamp, now) + } +} + +// readRaids emits the distinct active raid options, pruning expired keys. +// Strong Range (not RangeRelaxed): each key visited at most once. +func readRaids(now int64) []ApiGymRaidAvailable { + out := []ApiGymRaidAvailable{} + raidExpiry.Range(func(k raidKey, exp int64) bool { + if exp > now { + out = append(out, ApiGymRaidAvailable{RaidLevel: k.RaidLevel, PokemonId: k.PokemonId, Form: k.Form}) + } else { + pruneExpired(raidExpiry, k, now) + } + return true + }) + return out +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `go test ./decoder/ -run TestObserveExpiryAndReadRaids -count=1` +Expected: PASS. + +- [ ] **Step 5: Wire init + the gym hook** — `decoder/fortRtree.go` + +In `initFortRtree`, add `initFortAvailability()` next to the existing `initQuestConditions()` call: + +```go + initQuestConditions() + initFortAvailability() +``` + +Rewrite `updateGymLookup` to build the lookup as a value, store it, then observe (mirrors how updateStationLookupWithBattles builds a `lookup` var): + +```go +func updateGymLookup(gym *Gym) { + now := time.Now().Unix() + fl := FortLookup{ + FortType: GYM, + Lat: gym.Lat, + Lon: gym.Lon, + IsArScanEligible: gym.ArScanEligible.ValueOrZero() == 1, + AvailableSlots: int8(gym.AvailableSlots.ValueOrZero()), + TeamId: int8(gym.TeamId.ValueOrZero()), + RaidEndTimestamp: gym.RaidEndTimestamp.ValueOrZero(), + RaidBattleTimestamp: gym.RaidBattleTimestamp.ValueOrZero(), + RaidLevel: int8(gym.RaidLevel.ValueOrZero()), + RaidPokemonId: int16(gym.RaidPokemonId.ValueOrZero()), + RaidPokemonForm: int16(gym.RaidPokemonForm.ValueOrZero()), + } + fortLookupCache.Store(gym.Id, fl) + observeRaid(&fl, now) +} +``` + +Confirm `time` is already imported in `fortRtree.go` (it is — `updateStationLookup` uses `time.Now()`). + +- [ ] **Step 6: Point `GetAvailableGyms` at the map + drop the accumulator and Count** — `decoder/api_gym_available.go` + +Replace everything from the `ApiGymRaidAvailable` struct through `GetAvailableGyms` with: + +```go +// ApiGymRaidAvailable is one distinct active raid option on resident gyms. +// PokemonId 0 means an egg (no boss yet). ReactMap derives its e/r/boss keys. +type ApiGymRaidAvailable struct { + RaidLevel int8 `json:"raid_level" doc:"Raid level/tier"` + PokemonId int16 `json:"pokemon_id" doc:"Raid boss pokemon id; 0 = egg (unhatched)"` + Form int16 `json:"form" doc:"Raid boss form id, else 0"` +} + +// ApiAvailableGyms is the whole-instance gym filter snapshot. Only raids are +// dynamic — team/slot filter keys are generated statically by the consumer, so +// they are not aggregated here. +type ApiAvailableGyms struct { + Raids []ApiGymRaidAvailable `json:"raids" doc:"Distinct active raid levels/bosses/eggs on resident gyms"` +} + +// GetAvailableGyms reads the maintained raid index (no fort scan). +func GetAvailableGyms(now int64) *ApiAvailableGyms { + res := &ApiAvailableGyms{Raids: readRaids(now)} + log.Infof("available-gyms: %d raid options (maintained)", len(res.Raids)) + return res +} +``` + +Delete the now-unused `gymAvailAcc`, `newGymAvailAcc`, `ingest`, `result`. Remove the `"time"` import from `api_gym_available.go` if it is now unused (it is — no more `time.Now`/`time.Since`); keep `log`. + +- [ ] **Step 7: Rewrite the gym availability test** — `decoder/api_gym_available_test.go` + +Replace `TestGetAvailableGyms` (which seeded `fortLookupCache`) with one that seeds via the hook: + +```go +func TestGetAvailableGyms(t *testing.T) { + initFortAvailability() + now := int64(1_000_000) + + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: now + 100}, now) // boss + observeRaid(&FortLookup{RaidLevel: 3, RaidPokemonId: 0, RaidEndTimestamp: now + 100}, now) // egg + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 999, RaidEndTimestamp: now - 1}, now) // expired -> ignored + + res := GetAvailableGyms(now) + + var bosses, eggs int + for _, r := range res.Raids { + if r.PokemonId == 999 { + t.Fatalf("expired raid leaked: %+v", r) + } + if r.PokemonId == 0 { + eggs++ + } else { + bosses++ + } + } + if bosses != 1 || eggs != 1 { + t.Fatalf("raids: %+v", res.Raids) + } +} +``` + +- [ ] **Step 8: Build + full decoder test** + +Run: `go build -tags go_json ./... && go test ./decoder/ -count=1` +Expected: build OK; all tests pass. (The combined test `TestGetAvailableForts` still passes here because `GetAvailableForts` is untouched until Task 4 — but note it now depends on `GetAvailableGyms` reading the map; if it seeds `fortLookupCache` it will show 0 raids. If `TestGetAvailableForts` fails on the gyms section, that is expected and is fixed in Task 4 — leave it; do NOT weaken it here. If it fails, note it in the commit and proceed.) + +Actually, to avoid a red suite between tasks, update the gyms assertion in `TestGetAvailableForts` now only if it references `combined.Gyms.Raids` counts that the map won't have. Prefer: in Task 4 the combined test is rewritten to seed via observe. If Step 8's suite is red solely due to `TestGetAvailableForts`, temporarily skip that one test with `t.Skip("rebuilt in Task 4: combined reads maintained maps")` and remove the skip in Task 4. + +- [ ] **Step 9: Commit** + +```bash +git add decoder/fort_availability.go decoder/fort_availability_test.go decoder/fortRtree.go decoder/api_gym_available.go decoder/api_gym_available_test.go +git commit +``` +Message: `feat(availability): maintained raid index + observe/prune primitive` + +--- + +## Task 2: Station battles + +**Files:** +- Modify: `decoder/fort_availability.go` (add `battleExpiry` observe/read) +- Modify: `decoder/fort_availability_test.go` (add battle test) +- Modify: `decoder/fortRtree.go` (`updateStationLookupWithBattles`) +- Modify: `decoder/api_station_available.go` +- Modify: `decoder/api_station_available_test.go` + +**Interfaces:** +- Consumes (Task 1): `observeExpiry`, `pruneExpired`, `battleExpiry`, `type battleKey`, `initFortAvailability`. +- Produces: `func observeStationBattles(fl *FortLookup, now int64)`, `func readBattles(now int64) []ApiStationBattleAvailable`. + +- [ ] **Step 1: Write the failing test** — append to `decoder/fort_availability_test.go` + +```go +func TestObserveStationBattlesAndRead(t *testing.T) { + initFortAvailability() + now := int64(1000) + + // station with two active battles (slice) — both distinct options + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ + {BattleLevel: 5, BattlePokemonId: 150, BattlePokemonForm: 0, BattleEndTimestamp: 2000}, + {BattleLevel: 3, BattlePokemonId: 0, BattlePokemonForm: 0, BattleEndTimestamp: 2000}, + }}, now) + // level 0 -> ignored; expired -> ignored + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ + {BattleLevel: 0, BattleEndTimestamp: 2000}, + {BattleLevel: 5, BattlePokemonId: 999, BattleEndTimestamp: 500}, + }}, now) + // no slice: fall back to the top-battle scalar projection + observeStationBattles(&FortLookup{BattleLevel: 4, BattlePokemonId: 200, BattleEndTimestamp: 2000}, now) + + got := readBattles(now) + if len(got) != 3 { + t.Fatalf("want 3 battle options, got %d: %+v", len(got), got) + } + for _, b := range got { + if b.PokemonId == 999 { + t.Fatal("expired battle leaked") + } + } + if len(readBattles(3000)) != 0 { + t.Fatal("all battles expired -> empty") + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./decoder/ -run TestObserveStationBattlesAndRead -count=1` +Expected: FAIL — `undefined: observeStationBattles` / `readBattles`. + +- [ ] **Step 3: Add battle observe/read** — `decoder/fort_availability.go` + +```go +func observeStationBattles(fl *FortLookup, now int64) { + obs := func(level int8, id, form int16, end int64) { + if level == 0 { + return + } + observeExpiry(battleExpiry, battleKey{level, id, form}, end, now) + } + if len(fl.StationBattles) == 0 { + obs(fl.BattleLevel, fl.BattlePokemonId, fl.BattlePokemonForm, fl.BattleEndTimestamp) + return + } + for _, b := range fl.StationBattles { + obs(b.BattleLevel, b.BattlePokemonId, b.BattlePokemonForm, b.BattleEndTimestamp) + } +} + +func readBattles(now int64) []ApiStationBattleAvailable { + out := []ApiStationBattleAvailable{} + battleExpiry.Range(func(k battleKey, exp int64) bool { + if exp > now { + out = append(out, ApiStationBattleAvailable{BattleLevel: k.BattleLevel, PokemonId: k.PokemonId, Form: k.Form}) + } else { + pruneExpired(battleExpiry, k, now) + } + return true + }) + return out +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `go test ./decoder/ -run TestObserveStationBattlesAndRead -count=1` +Expected: PASS. + +- [ ] **Step 5: Add the station hook** — `decoder/fortRtree.go` + +In `updateStationLookupWithBattles`, after the `fortLookupCache.Store(station.Id, lookup)` line, add: + +```go + fortLookupCache.Store(station.Id, lookup) + observeStationBattles(&lookup, time.Now().Unix()) +``` + +(`applyTopStationBattleToFortLookup` has already populated `lookup`'s scalar top-battle fields, so the no-slice fallback in `observeStationBattles` sees them.) + +- [ ] **Step 6: Point `GetAvailableStations` at the map + drop the accumulator and Count** — `decoder/api_station_available.go` + +Replace from the `ApiStationBattleAvailable` struct through `GetAvailableStations` with: + +```go +// ApiStationBattleAvailable is one distinct active (battle_level, pokemon, form) +// option on resident stations. ReactMap derives its - and j keys. +type ApiStationBattleAvailable struct { + BattleLevel int8 `json:"battle_level" doc:"Max battle level"` + PokemonId int16 `json:"pokemon_id" doc:"Battle pokemon id, else 0"` + Form int16 `json:"form" doc:"Battle pokemon form id, else 0"` +} + +// ApiAvailableStations is the whole-instance station filter snapshot served by +// GET /api/station/available. +type ApiAvailableStations struct { + Battles []ApiStationBattleAvailable `json:"battles" doc:"Distinct active battle level/pokemon options on resident stations"` +} + +// GetAvailableStations reads the maintained battle index (no fort scan). +func GetAvailableStations(now int64) *ApiAvailableStations { + res := &ApiAvailableStations{Battles: readBattles(now)} + log.Infof("available-stations: %d battle options (maintained)", len(res.Battles)) + return res +} +``` + +Delete `stationAvailAcc`, `newStationAvailAcc`, `add`, `ingest`, `result`. Remove the now-unused `"time"` import; keep `log`. + +- [ ] **Step 7: Rewrite the station availability test** — `decoder/api_station_available_test.go` + +Replace the body that seeded `fortLookupCache` with observe-based seeding (mirror Task 1 Step 7): `initFortAvailability()`, `observeStationBattles(&FortLookup{...}, now)` for a live battle and an expired one, then assert `GetAvailableStations(now).Battles` contains only the live option. Keep the same level/pokemon values the old test used. + +- [ ] **Step 8: Build + full decoder test** + +Run: `go build -tags go_json ./... && go test ./decoder/ -count=1` +Expected: build OK; tests pass (the `TestGetAvailableForts` skip from Task 1 still in place). + +- [ ] **Step 9: Commit** + +```bash +git add decoder/fort_availability.go decoder/fort_availability_test.go decoder/fortRtree.go decoder/api_station_available.go decoder/api_station_available_test.go +git commit +``` +Message: `feat(availability): maintained station battle index` + +--- + +## Task 3: Pokestop lures, showcases, invasions + +**Files:** +- Modify: `decoder/fort_availability.go` (add lure/showcase/invasion observe/read) +- Modify: `decoder/fort_availability_test.go` +- Modify: `decoder/fortRtree.go` (`updatePokestopLookup`, `updatePokestopIncidentLookup`) +- Modify: `decoder/api_pokestop_available.go` +- Modify/Create: `decoder/api_pokestop_available_test.go` + +**Interfaces:** +- Consumes (Task 1): `observeExpiry`, `pruneExpired`, `lureExpiry`, `showcaseExpiry`, `invasionExpiry`, `showcaseKey`, `invasionKey`. +- Produces: `func observePokestop(fl *FortLookup, now int64)` (lure + showcase), `func observeInvasion(inc *FortLookupIncident, now int64)`, `func readLures/readShowcases/readInvasions(now int64) []Api...`. + +- [ ] **Step 1: Write the failing test** — append to `decoder/fort_availability_test.go` + +```go +func TestObservePokestopAggregatesAndRead(t *testing.T) { + initFortAvailability() + now := int64(1000) + + // lure + showcase on one stop + observePokestop(&FortLookup{ + LureId: 501, LureExpireTimestamp: 2000, + ContestPokemonId: 25, ContestPokemonForm: 0, ContestPokemonType: 0, ShowcaseExpiry: 2000, + }, now) + // expired lure + no showcase -> both ignored + observePokestop(&FortLookup{LureId: 502, LureExpireTimestamp: 500}, now) + + // invasions (per incident) + observeInvasion(&FortLookupIncident{Character: 5, DisplayType: 1, Confirmed: true, Slot1PokemonId: 41, ExpireTimestamp: 2000}, now) + observeInvasion(&FortLookupIncident{DisplayType: 9, ExpireTimestamp: 2000}, now) // showcase incident, character 0 + observeInvasion(&FortLookupIncident{Character: 30, DisplayType: 3, ExpireTimestamp: 500}, now) // expired + + if l := readLures(now); len(l) != 1 || l[0].LureId != 501 { + t.Fatalf("lures: %+v", l) + } + if s := readShowcases(now); len(s) != 1 || s[0].PokemonId != 25 { + t.Fatalf("showcases: %+v", s) + } + inv := readInvasions(now) + if len(inv) != 2 { + t.Fatalf("want 2 invasions, got %d: %+v", len(inv), inv) + } + for _, in := range inv { + if in.Character == 30 { + t.Fatal("expired invasion leaked") + } + } + // everything expires + if len(readLures(3000)) != 0 || len(readShowcases(3000)) != 0 || len(readInvasions(3000)) != 0 { + t.Fatal("all pokestop aggregates should expire to empty") + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./decoder/ -run TestObservePokestopAggregatesAndRead -count=1` +Expected: FAIL — undefined `observePokestop` / `observeInvasion` / `readLures` / `readShowcases` / `readInvasions`. + +- [ ] **Step 3: Add pokestop observe/read** — `decoder/fort_availability.go` + +```go +func observePokestop(fl *FortLookup, now int64) { + if fl.LureId != 0 { + observeExpiry(lureExpiry, fl.LureId, fl.LureExpireTimestamp, now) + } + if fl.ContestPokemonId != 0 { + observeExpiry(showcaseExpiry, showcaseKey{fl.ContestPokemonId, fl.ContestPokemonForm, fl.ContestPokemonType}, fl.ShowcaseExpiry, now) + } +} + +func observeInvasion(inc *FortLookupIncident, now int64) { + observeExpiry(invasionExpiry, invasionKey{ + Character: inc.Character, DisplayType: int16(inc.DisplayType), Confirmed: inc.Confirmed, + Slot1PokemonId: inc.Slot1PokemonId, Slot1Form: inc.Slot1Form, + }, inc.ExpireTimestamp, now) +} + +func readLures(now int64) []ApiPokestopLureAvailable { + out := []ApiPokestopLureAvailable{} + lureExpiry.Range(func(k int16, exp int64) bool { + if exp > now { + out = append(out, ApiPokestopLureAvailable{LureId: k}) + } else { + pruneExpired(lureExpiry, k, now) + } + return true + }) + return out +} + +func readShowcases(now int64) []ApiPokestopShowcaseAvailable { + out := []ApiPokestopShowcaseAvailable{} + showcaseExpiry.Range(func(k showcaseKey, exp int64) bool { + if exp > now { + out = append(out, ApiPokestopShowcaseAvailable{PokemonId: k.PokemonId, Form: k.Form, TypeId: k.TypeId}) + } else { + pruneExpired(showcaseExpiry, k, now) + } + return true + }) + return out +} + +func readInvasions(now int64) []ApiPokestopInvasionAvailable { + out := []ApiPokestopInvasionAvailable{} + invasionExpiry.Range(func(k invasionKey, exp int64) bool { + if exp > now { + out = append(out, ApiPokestopInvasionAvailable{ + Character: k.Character, DisplayType: k.DisplayType, Confirmed: k.Confirmed, + Slot1PokemonId: k.Slot1PokemonId, Slot1Form: k.Slot1Form, + }) + } else { + pruneExpired(invasionExpiry, k, now) + } + return true + }) + return out +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `go test ./decoder/ -run TestObservePokestopAggregatesAndRead -count=1` +Expected: PASS. + +- [ ] **Step 5: Add the pokestop hooks** — `decoder/fortRtree.go` + +In `updatePokestopLookup`, after the `Compute` block that stores the FortLookup and before `reconcileFortQuestConditions(...)`, observe lure + showcase from the freshly built values. The function already has `pokestop`; add a `now` and observe: + +```go + }) + + observePokestop(&FortLookup{ + LureId: pokestop.LureId, + LureExpireTimestamp: pokestop.LureExpireTimestamp.ValueOrZero(), + ContestPokemonId: int16(pokestop.ShowcasePokemon.ValueOrZero()), + ContestPokemonForm: int16(pokestop.ShowcasePokemonForm.ValueOrZero()), + ContestPokemonType: int8(pokestop.ShowcasePokemonType.ValueOrZero()), + ShowcaseExpiry: pokestop.ShowcaseExpiry.ValueOrZero(), + }, time.Now().Unix()) + + // This is the sole writer of a pokestop's FortLookup entry ... + reconcileFortQuestConditions(pokestop.Id, questConditionKeysFromPokestop(pokestop)) +``` + +In `updatePokestopIncidentLookup`, after the `updated := FortLookupIncident{...}` is built (before or after the Compute), observe it — it already has `now`: + +```go + observeExpiry(invasionExpiry, invasionKey{ + Character: updated.Character, DisplayType: int16(updated.DisplayType), Confirmed: updated.Confirmed, + Slot1PokemonId: updated.Slot1PokemonId, Slot1Form: updated.Slot1Form, + }, updated.ExpireTimestamp, now) +``` + +Prefer calling the helper: `observeInvasion(&updated, now)`. + +- [ ] **Step 6: Point `GetAvailablePokestops` at the maps + delete the scan machinery** — `decoder/api_pokestop_available.go` + +Drop `Count` from `ApiPokestopInvasionAvailable`, `ApiPokestopLureAvailable`, `ApiPokestopShowcaseAvailable` (delete each `Count int ...` field line). Leave `ApiPokestopQuestAvailable` and `ApiQuestConditionResult` untouched (quests keep counts). + +Replace `pokestopAvailAcc`, `newPokestopAvailAcc`, its `ingest`, its `result`, `GetAvailablePokestops`, `questRewardKey`, `verifyQuestAggregate`, and `logAvailablePokestops` with: + +```go +// GetAvailablePokestops reads the maintained lure/showcase/invasion indexes and +// the maintained quest-conditions aggregate (quests unchanged) — no fort scan. +func GetAvailablePokestops(now int64) *ApiAvailablePokestops { + res := &ApiAvailablePokestops{ + Quests: []ApiPokestopQuestAvailable{}, + Invasions: readInvasions(now), + Lures: readLures(now), + Showcases: readShowcases(now), + } + for _, c := range GetAvailableQuestConditions() { + res.Quests = append(res.Quests, ApiPokestopQuestAvailable(c)) + } + log.Infof("available-pokestops: %d quests, %d invasions, %d lures, %d showcases (maintained)", + len(res.Quests), len(res.Invasions), len(res.Lures), len(res.Showcases)) + return res +} +``` + +Keep the `ApiAvailablePokestops` struct and the four `Api*Available` types (minus Count). Remove the now-unused `"time"` import if unused; keep `log`. If `questRewardKey` is referenced nowhere else (confirm with `grep -rn questRewardKey decoder/`), delete it. + +- [ ] **Step 7: Pokestop availability test** — `decoder/api_pokestop_available_test.go` + +If a pokestop-available test exists, rewrite it to seed via `observePokestop` / `observeInvasion` + a seeded quest condition, then assert `GetAvailablePokestops(now)` sections. If none exists, the coverage in `fort_availability_test.go` Step 1 plus the endpoint smoke in Task 4 suffice — do not invent a redundant one. Verify no test still references the deleted `verifyQuestAggregate` / `pokestopAvailAcc` (grep; fix any). + +- [ ] **Step 8: Build + full decoder test** + +Run: `go build -tags go_json ./... && go test ./decoder/ -count=1` +Expected: build OK; tests pass. + +- [ ] **Step 9: Commit** + +```bash +git add decoder/fort_availability.go decoder/fort_availability_test.go decoder/fortRtree.go decoder/api_pokestop_available.go decoder/api_pokestop_available_test.go +git commit +``` +Message: `feat(availability): maintained pokestop lure/showcase/invasion indexes` + +--- + +## Task 4: Combined endpoint reads maps; final scan removal + +**Files:** +- Modify: `decoder/api_fort_available.go` +- Modify: `decoder/api_fort_available_test.go` + +**Interfaces:** +- Consumes: `GetAvailablePokestops`, `GetAvailableGyms`, `GetAvailableStations` (all now map-backed). + +- [ ] **Step 1: Rewrite `TestGetAvailableForts`** — `decoder/api_fort_available_test.go` + +Remove any `t.Skip` added in Task 1. Seed the maintained maps via observe, then assert the combined result assembles all three sections: + +```go +func TestGetAvailableForts(t *testing.T) { + initFortAvailability() + initQuestConditions() + now := int64(1_000_000) + + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: now + 100}, now) + observePokestop(&FortLookup{LureId: 501, LureExpireTimestamp: now + 100}, now) + observeInvasion(&FortLookupIncident{Character: 5, DisplayType: 1, ExpireTimestamp: now + 100}, now) + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ + {BattleLevel: 5, BattlePokemonId: 150, BattleEndTimestamp: now + 100}, + }}, now) + + combined := GetAvailableForts(now) + if len(combined.Gyms.Raids) != 1 { + t.Fatalf("gyms: %+v", combined.Gyms) + } + if len(combined.Pokestops.Lures) != 1 || len(combined.Pokestops.Invasions) != 1 { + t.Fatalf("pokestops: %+v", combined.Pokestops) + } + if len(combined.Stations.Battles) != 1 { + t.Fatalf("stations: %+v", combined.Stations) + } + + // parity: combined sections equal the per-type reads over the same maps + if len(GetAvailableGyms(now).Raids) != len(combined.Gyms.Raids) || + len(GetAvailableStations(now).Battles) != len(combined.Stations.Battles) { + t.Fatal("combined diverges from per-type builders") + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./decoder/ -run TestGetAvailableForts -count=1` +Expected: FAIL — `GetAvailableForts` still does `fortLookupCache.Range` and returns empty sections (maps are seeded, cache is not). + +- [ ] **Step 3: Rewrite `GetAvailableForts`** — `decoder/api_fort_available.go` + +```go +// GetAvailableForts assembles all three availability sections from the +// maintained indexes — no fortLookupCache scan. +func GetAvailableForts(now int64) *ApiAvailableForts { + res := &ApiAvailableForts{ + Pokestops: GetAvailablePokestops(now), + Gyms: GetAvailableGyms(now), + Stations: GetAvailableStations(now), + } + log.Infof("available-forts: %d raid, %d lure, %d invasion, %d showcase, %d battle options (maintained)", + len(res.Gyms.Raids), len(res.Pokestops.Lures), len(res.Pokestops.Invasions), + len(res.Pokestops.Showcases), len(res.Stations.Battles)) + return res +} +``` + +Update the `ApiAvailableForts` doc comment to drop "one fortLookupCache range". Remove the now-unused `"time"` import; keep `log`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `go test ./decoder/ -run TestGetAvailableForts -count=1` +Expected: PASS. + +- [ ] **Step 5: Assert the scan is gone** + +Run: `grep -rn "fortLookupCache.Range" decoder/api_*_available.go decoder/api_fort_available.go` +Expected: no matches (the availability path no longer scans). If any remain, they are a leftover to remove. + +- [ ] **Step 6: Build + full decoder + route tests** + +Run: `go build -tags go_json ./... && go test ./decoder/ -count=1 && go test . -count=1` +Expected: all green (route/huma tests for the availability endpoints still register and 503-gate correctly; the response shapes changed only by dropping `count`). + +- [ ] **Step 7: Commit** + +```bash +git add decoder/api_fort_available.go decoder/api_fort_available_test.go +git commit +``` +Message: `feat(availability): combined endpoint reads maintained indexes; drop fort scan` + +--- + +## Self-Review + +**Spec coverage:** +- §3 five maps + observe primitive → Task 1 (primitive + raid), Tasks 2-3 (battle, lure/showcase/invasion). ✅ +- §3.1 `observeExpiry` (ignore expired, keep-larger) → Task 1 Step 3, tested Step 1. ✅ +- §3.2 hooks in the four update fns → Task 1 Step 5 (gym), Task 2 Step 5 (station), Task 3 Step 5 (pokestop + incident). ✅ +- §3.3 prune-on-read conditional delete, strong Range → `pruneExpired` (Task 1 Step 3), used by every `read*`. ✅ +- §4 removals (scan, accumulators, verifyQuestAggregate, Count) → Tasks 1/2/3 drop per-type accs + Count; Task 3 drops verifyQuestAggregate; Task 4 drops the combined scan and asserts none remain (Step 5). ✅ +- §3 init warms at startup → `initFortAvailability` in `initFortRtree`; hooks fire during preload because preload calls the same update fns (unchanged behavior). ✅ +- §8 drop Count, quests keep counts → Tasks 1/2/3 drop Count on the five dynamic structs; `ApiPokestopQuestAvailable`/`ApiQuestConditionResult` untouched. ✅ +- §9 tests → each task TDDs its aggregate; Task 4 covers the combined + a no-scan grep assertion. ✅ +- §10 non-goals (quests, ReactMap, pokemon, no sweep) → honored; no ReactMap or pokemon file touched, no sweep added. ✅ + +**Placeholder scan:** No TBD/TODO; every code step shows complete code. The only conditional instruction (Task 3 Step 7 "if a pokestop-available test exists") resolves to a concrete grep-and-decide, not a placeholder. + +**Type consistency:** `raidKey`/`battleKey`/`showcaseKey`/`invasionKey` field names are consistent between the key structs (Task 1 Step 3), the observe functions, and the read functions. `observeExpiry`/`pruneExpired` signatures match all call sites. `ApiGymRaidAvailable`/`ApiStationBattleAvailable`/`ApiPokestop*Available` field names match their (Count-stripped) struct definitions. From 410bb011052c6319792d88b27d483c44e79c6796 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 21:10:44 +0100 Subject: [PATCH 15/29] feat(availability): maintained raid index + observe/prune primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the maintained max-expiry availability index primitive (observeExpiry/pruneExpired on xsync.Map, keep-larger, ignore-expired, prune-on-read via conditional Compute) and wire the first consumer: gym raids. updateGymLookup now observes each gym's raid into raidExpiry on save; GetAvailableGyms reads readRaids(now) instead of scanning fortLookupCache, and drops the per-option Count field. GetAvailableForts (decoder/api_fort_available.go) is adjusted only as far as needed to keep building: its gyms section now calls GetAvailableGyms(now) instead of the deleted gym accumulator; pokestops and stations still scan fortLookupCache as before (migrated in Tasks 2-4). TestGetAvailableForts is skipped with a note — it seeds fortLookupCache directly so the maintained raidExpiry map stays empty; it's rebuilt to seed via observe in Task 4. Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available.go | 18 ++--- decoder/api_fort_available_test.go | 1 + decoder/api_gym_available.go | 60 ++--------------- decoder/api_gym_available_test.go | 24 ++----- decoder/fortRtree.go | 8 ++- decoder/fort_availability.go | 102 +++++++++++++++++++++++++++++ decoder/fort_availability_test.go | 42 ++++++++++++ 7 files changed, 170 insertions(+), 85 deletions(-) create mode 100644 decoder/fort_availability.go create mode 100644 decoder/fort_availability_test.go diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go index 11abfa46..9537ade9 100644 --- a/decoder/api_fort_available.go +++ b/decoder/api_fort_available.go @@ -16,15 +16,17 @@ type ApiAvailableForts struct { Stations *ApiAvailableStations `json:"stations" doc:"Station availability (same shape as /api/station/available)"` } -// GetAvailableForts builds all three availability aggregates in a single -// fortLookupCache range, dispatching each fort to its type's accumulator. +// GetAvailableForts builds the pokestop/station availability aggregates in a +// single fortLookupCache range, dispatching each fort to its type's +// accumulator. Gyms are read from the maintained raid index (see +// decoder/fort_availability.go) instead of being scanned here; Tasks 2-4 +// migrate pokestops/stations onto the same maintained-index pattern and drop +// this scan entirely. func GetAvailableForts(now int64) *ApiAvailableForts { start := time.Now() - g, p, s := newGymAvailAcc(), newPokestopAvailAcc(), newStationAvailAcc() + p, s := newPokestopAvailAcc(), newStationAvailAcc() fortLookupCache.Range(func(_ string, fl FortLookup) bool { switch fl.FortType { - case GYM: - g.ingest(&fl, now) case POKESTOP: p.ingest(&fl, now) case STATION: @@ -36,14 +38,14 @@ func GetAvailableForts(now int64) *ApiAvailableForts { // per-type finalizer. res := &ApiAvailableForts{ Pokestops: p.result(), - Gyms: g.result(), + Gyms: GetAvailableGyms(now), Stations: s.result(), } verifyQuestAggregate(p.rewards) // same pokestop cross-check the per-type build runs if statsCollector != nil { statsCollector.ObserveApiScan("available-forts", time.Since(start).Seconds()) } - log.Infof("available-forts built in %s: one pass over %d gyms / %d pokestops / %d stations", - time.Since(start), g.forts, p.forts, s.forts) + log.Infof("available-forts built in %s: one pass over %d pokestops / %d stations (gyms maintained)", + time.Since(start), p.forts, s.forts) return res } diff --git a/decoder/api_fort_available_test.go b/decoder/api_fort_available_test.go index d53fac35..908c126c 100644 --- a/decoder/api_fort_available_test.go +++ b/decoder/api_fort_available_test.go @@ -9,6 +9,7 @@ import ( // TestGetAvailableForts locks that the single-pass combined builder produces // the same aggregates as the three per-type builders over the same cache. func TestGetAvailableForts(t *testing.T) { + t.Skip("rebuilt in Task 4: combined reads maintained maps") fortLookupCache = xsync.NewMap[string, FortLookup]() initQuestConditions() now := int64(1_000_000) diff --git a/decoder/api_gym_available.go b/decoder/api_gym_available.go index a2fb0500..a7fdc7f3 100644 --- a/decoder/api_gym_available.go +++ b/decoder/api_gym_available.go @@ -1,8 +1,6 @@ package decoder import ( - "time" - log "github.com/sirupsen/logrus" ) @@ -12,66 +10,18 @@ type ApiGymRaidAvailable struct { RaidLevel int8 `json:"raid_level" doc:"Raid level/tier"` PokemonId int16 `json:"pokemon_id" doc:"Raid boss pokemon id; 0 = egg (unhatched)"` Form int16 `json:"form" doc:"Raid boss form id, else 0"` - Count int `json:"count" doc:"Number of resident gyms with this raid option"` } -// ApiAvailableGyms is the whole-instance gym filter snapshot served by -// GET /api/gym/available. // ApiAvailableGyms is the whole-instance gym filter snapshot. Only raids are -// dynamic — team/slot filter keys are generated statically by the consumer -// (every team/slot combination exists on a live instance), so they are not -// aggregated here. +// dynamic — team/slot filter keys are generated statically by the consumer, so +// they are not aggregated here. type ApiAvailableGyms struct { Raids []ApiGymRaidAvailable `json:"raids" doc:"Distinct active raid levels/bosses/eggs on resident gyms"` } -// GetAvailableGyms builds the gym filter snapshot from a single fortLookupCache -// range over resident gyms — no maintained map (FortLookup carries every gym -// filter field). Teams are all-resident (no time filter); raids require an -// unexpired raid with level > 0. -// gymAvailAcc accumulates the gym availability aggregate; ingest assumes the -// fort is a GYM. Shared by the per-type and combined builders. -type gymAvailAcc struct { - raids map[ApiGymRaidAvailable]int - forts int -} - -func newGymAvailAcc() *gymAvailAcc { - return &gymAvailAcc{raids: map[ApiGymRaidAvailable]int{}} -} - -func (a *gymAvailAcc) ingest(fl *FortLookup, now int64) { - a.forts++ - if fl.RaidLevel > 0 && fl.RaidEndTimestamp > now { - a.raids[ApiGymRaidAvailable{RaidLevel: fl.RaidLevel, PokemonId: fl.RaidPokemonId, Form: fl.RaidPokemonForm}]++ - } -} - -// result is a pure finalizer — no logging (the caller owns the log line so the -// combined builder doesn't emit a spurious per-type "built" entry). -func (a *gymAvailAcc) result() *ApiAvailableGyms { - res := &ApiAvailableGyms{Raids: []ApiGymRaidAvailable{}} - for k, n := range a.raids { - k.Count = n - res.Raids = append(res.Raids, k) - } - return res -} - +// GetAvailableGyms reads the maintained raid index (no fort scan). func GetAvailableGyms(now int64) *ApiAvailableGyms { - start := time.Now() - acc := newGymAvailAcc() - fortLookupCache.Range(func(_ string, fl FortLookup) bool { - if fl.FortType == GYM { - acc.ingest(&fl, now) - } - return true - }) - res := acc.result() - if statsCollector != nil { - statsCollector.ObserveApiScan("available-gyms", time.Since(start).Seconds()) - } - log.Infof("available-gyms built in %s: scanned %d gyms -> %d raid options", - time.Since(start), acc.forts, len(res.Raids)) + res := &ApiAvailableGyms{Raids: readRaids(now)} + log.Infof("available-gyms: %d raid options (maintained)", len(res.Raids)) return res } diff --git a/decoder/api_gym_available_test.go b/decoder/api_gym_available_test.go index 7833abc6..240abf3a 100644 --- a/decoder/api_gym_available_test.go +++ b/decoder/api_gym_available_test.go @@ -2,34 +2,18 @@ package decoder import ( "testing" - - "github.com/puzpuzpuz/xsync/v4" ) func TestGetAvailableGyms(t *testing.T) { - fortLookupCache = xsync.NewMap[string, FortLookup]() + initFortAvailability() now := int64(1_000_000) - // gym with team + active raid boss - fortLookupCache.Store("g1", FortLookup{ - FortType: GYM, TeamId: 1, AvailableSlots: 2, - RaidLevel: 5, RaidPokemonId: 150, RaidPokemonForm: 0, RaidEndTimestamp: now + 100, - }) - // gym with an active egg (no boss) and an EXPIRED raid on another - fortLookupCache.Store("g2", FortLookup{ - FortType: GYM, TeamId: 2, AvailableSlots: 6, - RaidLevel: 3, RaidPokemonId: 0, RaidEndTimestamp: now + 100, - }) - fortLookupCache.Store("g3", FortLookup{ - FortType: GYM, TeamId: 1, AvailableSlots: 0, - RaidLevel: 5, RaidPokemonId: 999, RaidEndTimestamp: now - 1, // expired -> excluded - }) - // a pokestop must be ignored - fortLookupCache.Store("s1", FortLookup{FortType: POKESTOP, LureId: 501}) + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: now + 100}, now) // boss + observeRaid(&FortLookup{RaidLevel: 3, RaidPokemonId: 0, RaidEndTimestamp: now + 100}, now) // egg + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 999, RaidEndTimestamp: now - 1}, now) // expired -> ignored res := GetAvailableGyms(now) - // raids: boss 150 lvl5, egg lvl3; expired 999 excluded var bosses, eggs int for _, r := range res.Raids { if r.PokemonId == 999 { diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 188ae7fc..1de15da5 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -81,6 +81,7 @@ func initFortRtree() { fortTreeEvictor = newTreeEvictor[string]("fort", 65536, treeEvictorBatchSize, flushFortTreeEvictions) fortLookupCache = xsync.NewMap[string, FortLookup]() initQuestConditions() + initFortAvailability() // OnEviction registrations live here, after fortTreeEvictor and // fortLookupCache are created (and after pokestopCache/gymCache/ @@ -227,7 +228,8 @@ func updatePokestopLookup(pokestop *Pokestop) { } func updateGymLookup(gym *Gym) { - fortLookupCache.Store(gym.Id, FortLookup{ + now := time.Now().Unix() + fl := FortLookup{ FortType: GYM, Lat: gym.Lat, Lon: gym.Lon, @@ -239,7 +241,9 @@ func updateGymLookup(gym *Gym) { RaidLevel: int8(gym.RaidLevel.ValueOrZero()), RaidPokemonId: int16(gym.RaidPokemonId.ValueOrZero()), RaidPokemonForm: int16(gym.RaidPokemonForm.ValueOrZero()), - }) + } + fortLookupCache.Store(gym.Id, fl) + observeRaid(&fl, now) } func updateStationLookup(station *Station) { diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go new file mode 100644 index 00000000..4e252749 --- /dev/null +++ b/decoder/fort_availability.go @@ -0,0 +1,102 @@ +package decoder + +import "github.com/puzpuzpuz/xsync/v4" + +// Maintained max-expiry availability index. Each map holds, per distinct filter +// option, the latest expiry timestamp seen on any resident fort. Availability +// reads the maps instead of scanning fortLookupCache; see +// docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md. +// +// Quests are NOT here — they can be retracted mid-life (geofence clear, event +// swap) which a monotonic max-expiry cannot express, so they keep the reconcile +// aggregate (questConditionCount). + +type showcaseKey struct { + PokemonId int16 + Form int16 + TypeId int8 +} + +type invasionKey struct { + Character int16 + DisplayType int16 + Confirmed bool + Slot1PokemonId int16 + Slot1Form int16 +} + +type raidKey struct { + RaidLevel int8 + PokemonId int16 + Form int16 +} + +type battleKey struct { + BattleLevel int8 + PokemonId int16 + Form int16 +} + +var ( + lureExpiry *xsync.Map[int16, int64] + showcaseExpiry *xsync.Map[showcaseKey, int64] + invasionExpiry *xsync.Map[invasionKey, int64] + raidExpiry *xsync.Map[raidKey, int64] + battleExpiry *xsync.Map[battleKey, int64] +) + +func initFortAvailability() { + lureExpiry = xsync.NewMap[int16, int64]() + showcaseExpiry = xsync.NewMap[showcaseKey, int64]() + invasionExpiry = xsync.NewMap[invasionKey, int64]() + raidExpiry = xsync.NewMap[raidKey, int64]() + battleExpiry = xsync.NewMap[battleKey, int64]() +} + +// observeExpiry records key as available until at least expiry, keeping the +// larger of any prior expiry (a still-active fort refreshes the lifetime). +// Already-expired observations are ignored so dead keys never enter the map. +func observeExpiry[K comparable](m *xsync.Map[K, int64], key K, expiry, now int64) { + if expiry <= now { + return + } + m.Compute(key, func(old int64, _ bool) (int64, xsync.ComputeOp) { + if old >= expiry { + return old, xsync.CancelOp + } + return expiry, xsync.UpdateOp + }) +} + +// pruneExpired deletes key iff it is STILL expired. It must never be a blind +// Delete: that could race an observe that just refreshed the key and wrongly +// drop a live option. Compute re-checks under the key's lock. +func pruneExpired[K comparable](m *xsync.Map[K, int64], key K, now int64) { + m.Compute(key, func(cur int64, loaded bool) (int64, xsync.ComputeOp) { + if loaded && cur <= now { + return 0, xsync.DeleteOp + } + return cur, xsync.CancelOp + }) +} + +func observeRaid(fl *FortLookup, now int64) { + if fl.RaidLevel > 0 { + observeExpiry(raidExpiry, raidKey{fl.RaidLevel, fl.RaidPokemonId, fl.RaidPokemonForm}, fl.RaidEndTimestamp, now) + } +} + +// readRaids emits the distinct active raid options, pruning expired keys. +// Strong Range (not RangeRelaxed): each key visited at most once. +func readRaids(now int64) []ApiGymRaidAvailable { + out := []ApiGymRaidAvailable{} + raidExpiry.Range(func(k raidKey, exp int64) bool { + if exp > now { + out = append(out, ApiGymRaidAvailable{RaidLevel: k.RaidLevel, PokemonId: k.PokemonId, Form: k.Form}) + } else { + pruneExpired(raidExpiry, k, now) + } + return true + }) + return out +} diff --git a/decoder/fort_availability_test.go b/decoder/fort_availability_test.go new file mode 100644 index 00000000..6130079d --- /dev/null +++ b/decoder/fort_availability_test.go @@ -0,0 +1,42 @@ +package decoder + +import "testing" + +func TestObserveExpiryAndReadRaids(t *testing.T) { + initFortAvailability() + now := int64(1000) + + // active raid boss + active egg + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidPokemonForm: 0, RaidEndTimestamp: 2000}, now) + observeRaid(&FortLookup{RaidLevel: 3, RaidPokemonId: 0, RaidPokemonForm: 0, RaidEndTimestamp: 2000}, now) + // no raid (level 0) -> ignored + observeRaid(&FortLookup{RaidLevel: 0, RaidEndTimestamp: 2000}, now) + // already-expired -> ignored + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 999, RaidEndTimestamp: 500}, now) + + got := readRaids(now) + if len(got) != 2 { + t.Fatalf("want 2 raid options, got %d: %+v", len(got), got) + } + for _, r := range got { + if r.PokemonId == 999 { + t.Fatal("expired raid must not appear") + } + } + + // keep-larger: re-observe boss 150 with a LATER expiry, then read after the + // first expiry has passed — it must still be present. + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: 3000}, now) + if len(readRaids(2500)) == 0 { + t.Fatal("refreshed raid should survive past its first expiry") + } + + // prune-on-read: once fully expired, it drops out. + if len(readRaids(4000)) != 0 { + t.Fatal("all raids expired -> empty") + } + // and empty read returns [] not nil + if readRaids(4000) == nil { + t.Fatal("read must return non-nil empty slice") + } +} From 1a70b2dab454d42b5f238c60901ab89cf0727d7b Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 21:18:06 +0100 Subject: [PATCH 16/29] test(huma): drop stale teams assertion from gym available route test --- huma_routes_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/huma_routes_test.go b/huma_routes_test.go index 7613bfba..67ee3f94 100644 --- a/huma_routes_test.go +++ b/huma_routes_test.go @@ -544,9 +544,6 @@ func TestHumaGymAvailableRoute(t *testing.T) { if resp.Code != http.StatusOK { t.Fatalf("fim on: got %d, want 200; body=%s", resp.Code, resp.Body.String()) } - if !strings.Contains(resp.Body.String(), `"teams":[]`) { - t.Errorf("body missing \"teams\": %s", resp.Body.String()) - } if !strings.Contains(resp.Body.String(), `"raids":[]`) { t.Errorf("body missing \"raids\": %s", resp.Body.String()) } From 4fa9c5d5bffd4e5f13b5f82470d5ee2b9147f92c Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 21:25:06 +0100 Subject: [PATCH 17/29] feat(availability): maintained station battle index Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available.go | 22 ++++----- decoder/api_station_available.go | 64 ++------------------------- decoder/api_station_available_test.go | 20 ++++----- decoder/fortRtree.go | 1 + decoder/fort_availability.go | 35 +++++++++++++++ decoder/fort_availability_test.go | 31 +++++++++++++ 6 files changed, 88 insertions(+), 85 deletions(-) diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go index 9537ade9..264190ee 100644 --- a/decoder/api_fort_available.go +++ b/decoder/api_fort_available.go @@ -16,21 +16,17 @@ type ApiAvailableForts struct { Stations *ApiAvailableStations `json:"stations" doc:"Station availability (same shape as /api/station/available)"` } -// GetAvailableForts builds the pokestop/station availability aggregates in a -// single fortLookupCache range, dispatching each fort to its type's -// accumulator. Gyms are read from the maintained raid index (see -// decoder/fort_availability.go) instead of being scanned here; Tasks 2-4 -// migrate pokestops/stations onto the same maintained-index pattern and drop +// GetAvailableForts builds the pokestop availability aggregate from a single +// fortLookupCache range. Gyms and stations are read from their maintained +// indexes (see decoder/fort_availability.go) instead of being scanned here; +// Tasks 3-4 migrate pokestops onto the same maintained-index pattern and drop // this scan entirely. func GetAvailableForts(now int64) *ApiAvailableForts { start := time.Now() - p, s := newPokestopAvailAcc(), newStationAvailAcc() + p := newPokestopAvailAcc() fortLookupCache.Range(func(_ string, fl FortLookup) bool { - switch fl.FortType { - case POKESTOP: + if fl.FortType == POKESTOP { p.ingest(&fl, now) - case STATION: - s.ingest(&fl, now) } return true }) @@ -39,13 +35,13 @@ func GetAvailableForts(now int64) *ApiAvailableForts { res := &ApiAvailableForts{ Pokestops: p.result(), Gyms: GetAvailableGyms(now), - Stations: s.result(), + Stations: GetAvailableStations(now), } verifyQuestAggregate(p.rewards) // same pokestop cross-check the per-type build runs if statsCollector != nil { statsCollector.ObserveApiScan("available-forts", time.Since(start).Seconds()) } - log.Infof("available-forts built in %s: one pass over %d pokestops / %d stations (gyms maintained)", - time.Since(start), p.forts, s.forts) + log.Infof("available-forts built in %s: one pass over %d pokestops (gyms/stations maintained)", + time.Since(start), p.forts) return res } diff --git a/decoder/api_station_available.go b/decoder/api_station_available.go index 2be0bb16..d6dfef94 100644 --- a/decoder/api_station_available.go +++ b/decoder/api_station_available.go @@ -1,8 +1,6 @@ package decoder import ( - "time" - log "github.com/sirupsen/logrus" ) @@ -12,7 +10,6 @@ type ApiStationBattleAvailable struct { BattleLevel int8 `json:"battle_level" doc:"Max battle level"` PokemonId int16 `json:"pokemon_id" doc:"Battle pokemon id, else 0"` Form int16 `json:"form" doc:"Battle pokemon form id, else 0"` - Count int `json:"count" doc:"Number of resident stations with this active battle option"` } // ApiAvailableStations is the whole-instance station filter snapshot served by @@ -21,64 +18,9 @@ type ApiAvailableStations struct { Battles []ApiStationBattleAvailable `json:"battles" doc:"Distinct active battle level/pokemon options on resident stations"` } -// GetAvailableStations builds the station filter snapshot from a single -// fortLookupCache range. Mirrors isFortDnfMatch's station branch: iterate the -// StationBattles slice when present, else fall back to the top-battle -// projection; skip expired and level-0 battles. -// Unlike isFortDnfMatch, level-0 battles are excluded here (ReactMap's !battle_level convention). -// stationAvailAcc accumulates the station availability aggregate; ingest -// assumes the fort is a STATION. Shared by the per-type and combined builders. -type stationAvailAcc struct { - battles map[ApiStationBattleAvailable]int - forts int -} - -func newStationAvailAcc() *stationAvailAcc { - return &stationAvailAcc{battles: map[ApiStationBattleAvailable]int{}} -} - -func (a *stationAvailAcc) add(level int8, pokemonId, form int16, end, now int64) { - if level == 0 || end <= now { - return - } - a.battles[ApiStationBattleAvailable{BattleLevel: level, PokemonId: pokemonId, Form: form}]++ -} - -func (a *stationAvailAcc) ingest(fl *FortLookup, now int64) { - a.forts++ - if len(fl.StationBattles) == 0 { - a.add(fl.BattleLevel, fl.BattlePokemonId, fl.BattlePokemonForm, fl.BattleEndTimestamp, now) - return - } - for _, b := range fl.StationBattles { - a.add(b.BattleLevel, b.BattlePokemonId, b.BattlePokemonForm, b.BattleEndTimestamp, now) - } -} - -// result is a pure finalizer — no logging (see gymAvailAcc.result). -func (a *stationAvailAcc) result() *ApiAvailableStations { - res := &ApiAvailableStations{Battles: []ApiStationBattleAvailable{}} - for k, n := range a.battles { - k.Count = n - res.Battles = append(res.Battles, k) - } - return res -} - +// GetAvailableStations reads the maintained battle index (no fort scan). func GetAvailableStations(now int64) *ApiAvailableStations { - start := time.Now() - acc := newStationAvailAcc() - fortLookupCache.Range(func(_ string, fl FortLookup) bool { - if fl.FortType == STATION { - acc.ingest(&fl, now) - } - return true - }) - res := acc.result() - if statsCollector != nil { - statsCollector.ObserveApiScan("available-stations", time.Since(start).Seconds()) - } - log.Infof("available-stations built in %s: scanned %d stations -> %d battle options", - time.Since(start), acc.forts, len(res.Battles)) + res := &ApiAvailableStations{Battles: readBattles(now)} + log.Infof("available-stations: %d battle options (maintained)", len(res.Battles)) return res } diff --git a/decoder/api_station_available_test.go b/decoder/api_station_available_test.go index 4b0f0e97..949e6efb 100644 --- a/decoder/api_station_available_test.go +++ b/decoder/api_station_available_test.go @@ -2,32 +2,30 @@ package decoder import ( "testing" - - "github.com/puzpuzpuz/xsync/v4" ) func TestGetAvailableStations(t *testing.T) { - fortLookupCache = xsync.NewMap[string, FortLookup]() + initFortAvailability() now := int64(1_000_000) // station with two active battles (multi-battle path) + one expired - fortLookupCache.Store("st1", FortLookup{FortType: STATION, StationBattles: []FortLookupStationBattle{ + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ {BattleLevel: 3, BattlePokemonId: 150, BattlePokemonForm: 0, BattleEndTimestamp: now + 100}, {BattleLevel: 5, BattlePokemonId: 384, BattlePokemonForm: 0, BattleEndTimestamp: now + 100}, {BattleLevel: 1, BattlePokemonId: 1, BattleEndTimestamp: now - 1}, // expired -> excluded - }}) + }}, now) // station with only the top-battle projection (no StationBattles slice) - fortLookupCache.Store("st2", FortLookup{FortType: STATION, + observeStationBattles(&FortLookup{ BattleLevel: 6, BattlePokemonId: 999, BattlePokemonForm: 0, BattleEndTimestamp: now + 100, - }) + }, now) // station with a level-0 battle -> excluded - fortLookupCache.Store("st3", FortLookup{FortType: STATION, StationBattles: []FortLookupStationBattle{ + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ {BattleLevel: 0, BattlePokemonId: 5, BattleEndTimestamp: now + 100}, - }}) - fortLookupCache.Store("g1", FortLookup{FortType: GYM, TeamId: 1}) // ignored + }}, now) res := GetAvailableStations(now) - // expect: (3,150),(5,384) from st1, (6,999) from st2 = 3 distinct; expired + level-0 excluded + // expect: (3,150),(5,384) from the slice, (6,999) from the scalar projection = 3 + // distinct; expired + level-0 excluded if len(res.Battles) != 3 { t.Fatalf("battles: %+v", res.Battles) } diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 1de15da5..21e09e75 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -262,6 +262,7 @@ func updateStationLookupWithBattles(station *Station, stationBattles []StationBa } applyTopStationBattleToFortLookup(&lookup, stationBattles) fortLookupCache.Store(station.Id, lookup) + observeStationBattles(&lookup, time.Now().Unix()) } // updatePokestopIncidentLookup upserts the observed incident into a pokestop's FortLookup diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go index 4e252749..e5342cf9 100644 --- a/decoder/fort_availability.go +++ b/decoder/fort_availability.go @@ -100,3 +100,38 @@ func readRaids(now int64) []ApiGymRaidAvailable { }) return out } + +// observeStationBattles records every distinct active battle option on a +// station: the StationBattles slice when present, else the top-battle scalar +// projection. Level-0 ("no battle") observations are gated out here since +// observeExpiry only gates on expiry, not on the level-0 sentinel. +func observeStationBattles(fl *FortLookup, now int64) { + obs := func(level int8, id, form int16, end int64) { + if level == 0 { + return + } + observeExpiry(battleExpiry, battleKey{level, id, form}, end, now) + } + if len(fl.StationBattles) == 0 { + obs(fl.BattleLevel, fl.BattlePokemonId, fl.BattlePokemonForm, fl.BattleEndTimestamp) + return + } + for _, b := range fl.StationBattles { + obs(b.BattleLevel, b.BattlePokemonId, b.BattlePokemonForm, b.BattleEndTimestamp) + } +} + +// readBattles emits the distinct active station battle options, pruning +// expired keys. Strong Range (not RangeRelaxed): each key visited at most once. +func readBattles(now int64) []ApiStationBattleAvailable { + out := []ApiStationBattleAvailable{} + battleExpiry.Range(func(k battleKey, exp int64) bool { + if exp > now { + out = append(out, ApiStationBattleAvailable{BattleLevel: k.BattleLevel, PokemonId: k.PokemonId, Form: k.Form}) + } else { + pruneExpired(battleExpiry, k, now) + } + return true + }) + return out +} diff --git a/decoder/fort_availability_test.go b/decoder/fort_availability_test.go index 6130079d..2a6b7100 100644 --- a/decoder/fort_availability_test.go +++ b/decoder/fort_availability_test.go @@ -40,3 +40,34 @@ func TestObserveExpiryAndReadRaids(t *testing.T) { t.Fatal("read must return non-nil empty slice") } } + +func TestObserveStationBattlesAndRead(t *testing.T) { + initFortAvailability() + now := int64(1000) + + // station with two active battles (slice) — both distinct options + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ + {BattleLevel: 5, BattlePokemonId: 150, BattlePokemonForm: 0, BattleEndTimestamp: 2000}, + {BattleLevel: 3, BattlePokemonId: 0, BattlePokemonForm: 0, BattleEndTimestamp: 2000}, + }}, now) + // level 0 -> ignored; expired -> ignored + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ + {BattleLevel: 0, BattleEndTimestamp: 2000}, + {BattleLevel: 5, BattlePokemonId: 999, BattleEndTimestamp: 500}, + }}, now) + // no slice: fall back to the top-battle scalar projection + observeStationBattles(&FortLookup{BattleLevel: 4, BattlePokemonId: 200, BattleEndTimestamp: 2000}, now) + + got := readBattles(now) + if len(got) != 3 { + t.Fatalf("want 3 battle options, got %d: %+v", len(got), got) + } + for _, b := range got { + if b.PokemonId == 999 { + t.Fatal("expired battle leaked") + } + } + if len(readBattles(3000)) != 0 { + t.Fatal("all battles expired -> empty") + } +} From 17f605244c2ad201b0a76a0c6cbba02ece5a563a Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 21:40:57 +0100 Subject: [PATCH 18/29] feat(availability): maintained pokestop lure/showcase/invasion indexes Adds observePokestop/observeInvasion/readLures/readShowcases/readInvasions to the maintained max-expiry fort availability index and points GetAvailablePokestops at them, removing the pokestop full-fort scan (pokestopAvailAcc, verifyQuestAggregate, questRewardKey). Quests are untouched, still sourced from GetAvailableQuestConditions(). GetAvailableForts (decoder/api_fort_available.go) is also updated to call GetAvailablePokestops directly instead of the deleted accumulator, since otherwise the package would not compile; this is a minimal forward-compatible fix, not the full Task 4 rewrite. Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available.go | 27 +--- decoder/api_pokestop_available.go | 168 ++----------------------- decoder/api_pokestop_available_test.go | 30 ++--- decoder/fortRtree.go | 10 ++ decoder/fort_availability.go | 68 ++++++++++ decoder/fort_availability_test.go | 38 ++++++ 6 files changed, 145 insertions(+), 196 deletions(-) diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go index 264190ee..0f9cb4ca 100644 --- a/decoder/api_fort_available.go +++ b/decoder/api_fort_available.go @@ -7,41 +7,26 @@ import ( ) // ApiAvailableForts is the whole-instance availability snapshot for every fort -// type, served by GET /api/fort/available. One fortLookupCache range produces -// all three sections — on large instances this replaces three full-cache -// walks (one per per-type endpoint) with one. +// type, served by GET /api/fort/available. Each section is read from its own +// maintained index (see decoder/fort_availability.go) — no fortLookupCache scan. type ApiAvailableForts struct { Pokestops *ApiAvailablePokestops `json:"pokestops" doc:"Pokestop availability (same shape as /api/pokestop/available)"` Gyms *ApiAvailableGyms `json:"gyms" doc:"Gym availability (same shape as /api/gym/available)"` Stations *ApiAvailableStations `json:"stations" doc:"Station availability (same shape as /api/station/available)"` } -// GetAvailableForts builds the pokestop availability aggregate from a single -// fortLookupCache range. Gyms and stations are read from their maintained -// indexes (see decoder/fort_availability.go) instead of being scanned here; -// Tasks 3-4 migrate pokestops onto the same maintained-index pattern and drop -// this scan entirely. +// GetAvailableForts assembles all three availability sections from the +// maintained indexes — no fortLookupCache scan. func GetAvailableForts(now int64) *ApiAvailableForts { start := time.Now() - p := newPokestopAvailAcc() - fortLookupCache.Range(func(_ string, fl FortLookup) bool { - if fl.FortType == POKESTOP { - p.ingest(&fl, now) - } - return true - }) - // result() is pure — the combined builder emits ONE log line, not one per - // per-type finalizer. res := &ApiAvailableForts{ - Pokestops: p.result(), + Pokestops: GetAvailablePokestops(now), Gyms: GetAvailableGyms(now), Stations: GetAvailableStations(now), } - verifyQuestAggregate(p.rewards) // same pokestop cross-check the per-type build runs if statsCollector != nil { statsCollector.ObserveApiScan("available-forts", time.Since(start).Seconds()) } - log.Infof("available-forts built in %s: one pass over %d pokestops (gyms/stations maintained)", - time.Since(start), p.forts) + log.Infof("available-forts built in %s (pokestops/gyms/stations maintained)", time.Since(start)) return res } diff --git a/decoder/api_pokestop_available.go b/decoder/api_pokestop_available.go index c5ae835b..0a94b76e 100644 --- a/decoder/api_pokestop_available.go +++ b/decoder/api_pokestop_available.go @@ -1,8 +1,6 @@ package decoder import ( - "time" - log "github.com/sirupsen/logrus" ) @@ -31,23 +29,20 @@ type ApiPokestopInvasionAvailable struct { Confirmed bool `json:"confirmed" doc:"True when the lineup is confirmed (grunts only)"` Slot1PokemonId int16 `json:"slot1_pokemon_id" doc:"Confirmed lead pokemon id (grunts only), else 0"` Slot1Form int16 `json:"slot1_form" doc:"Confirmed lead pokemon form, else 0"` - Count int `json:"count" doc:"Number of resident forts with this active invasion signature"` } -// ApiPokestopLureAvailable is one distinct active lure type, with how many -// resident pokestops currently carry it. +// ApiPokestopLureAvailable is one distinct active lure type currently carried +// by resident pokestops. type ApiPokestopLureAvailable struct { LureId int16 `json:"lure_id" doc:"Active lure module id"` - Count int `json:"count" doc:"Number of resident forts with this active lure"` } // ApiPokestopShowcaseAvailable is one distinct active showcase contest -// (pokemon/form/type), with how many resident pokestops currently run it. +// (pokemon/form/type) currently run by resident pokestops. type ApiPokestopShowcaseAvailable struct { PokemonId int16 `json:"pokemon_id" doc:"Showcase focus pokemon id, else 0"` Form int16 `json:"form" doc:"Showcase focus pokemon form, else 0"` TypeId int8 `json:"type_id" doc:"Showcase focus pokemon type id (type-based showcases), else 0"` - Count int `json:"count" doc:"Number of resident forts with this active showcase"` } // ApiAvailablePokestops is the whole-instance snapshot served by @@ -59,160 +54,19 @@ type ApiAvailablePokestops struct { Showcases []ApiPokestopShowcaseAvailable `json:"showcases" doc:"Distinct active showcase focus pokemon/type"` } -// GetAvailablePokestops returns the distinct lures, active showcases, active -// invasions, and available quest options currently offered by resident -// pokestops, each with a count of how many forts carry it. Quests are sourced -// solely from the maintained quest-conditions map (Task 3); lures, showcases, -// and invasions come from a single fortLookupCache.Range pass. The same pass -// tallies FortLookup's own quest-reward fields and cross-checks that tally -// against the maintained map (verifyQuestAggregate) to catch reconciliation -// drift between the two. -// pokestopAvailAcc accumulates the pokestop availability aggregate; ingest -// assumes the fort is a POKESTOP. Shared by the per-type and combined builders. -type pokestopAvailAcc struct { - lures map[int16]int - shows map[ApiPokestopShowcaseAvailable]int // key without Count - inv map[ApiPokestopInvasionAvailable]int // key without Count - rewards map[questRewardKey]int // FortLookup reward tally — cross-checks the maintained map - forts, incidents int -} - -func newPokestopAvailAcc() *pokestopAvailAcc { - return &pokestopAvailAcc{ - lures: map[int16]int{}, - shows: map[ApiPokestopShowcaseAvailable]int{}, - inv: map[ApiPokestopInvasionAvailable]int{}, - rewards: map[questRewardKey]int{}, - } -} - -func (a *pokestopAvailAcc) ingest(fl *FortLookup, now int64) { - a.forts++ - if fl.LureId != 0 && fl.LureExpireTimestamp > now { - a.lures[fl.LureId]++ - } - if fl.ContestPokemonId != 0 && fl.ShowcaseExpiry > now { - a.shows[ApiPokestopShowcaseAvailable{PokemonId: fl.ContestPokemonId, Form: fl.ContestPokemonForm, TypeId: fl.ContestPokemonType}]++ - } - for _, in := range fl.Incidents { - if in.ExpireTimestamp <= now { - continue - } - a.incidents++ - a.inv[ApiPokestopInvasionAvailable{ - Character: in.Character, DisplayType: int16(in.DisplayType), Confirmed: in.Confirmed, - Slot1PokemonId: in.Slot1PokemonId, Slot1Form: in.Slot1Form, - }]++ - } - // FortLookup QuestNoAr* mirrors quest_* (the AR quest → with_ar=true); - // QuestAr* mirrors alternative_quest_* (non-AR → with_ar=false). Must - // match the questConditionKeysFromPokestop convention or the cross-check - // cries wolf. - if fl.QuestNoArRewardType != 0 { - a.rewards[questRewardKey{true, fl.QuestNoArRewardType, fl.QuestNoArRewardItemId, fl.QuestNoArRewardAmount, fl.QuestNoArRewardPokemonId, fl.QuestNoArRewardPokemonForm}]++ - } - if fl.QuestArRewardType != 0 { - a.rewards[questRewardKey{false, fl.QuestArRewardType, fl.QuestArRewardItemId, fl.QuestArRewardAmount, fl.QuestArRewardPokemonId, fl.QuestArRewardPokemonForm}]++ - } -} - -// result is a pure finalizer — no logging or verification (the caller owns -// those, so the combined builder doesn't emit a spurious per-type entry). -func (a *pokestopAvailAcc) result() *ApiAvailablePokestops { - // Initialize the slices so empty categories marshal as [] rather than null. +// GetAvailablePokestops reads the maintained lure/showcase/invasion indexes and +// the maintained quest-conditions aggregate (quests unchanged) — no fort scan. +func GetAvailablePokestops(now int64) *ApiAvailablePokestops { res := &ApiAvailablePokestops{ Quests: []ApiPokestopQuestAvailable{}, - Invasions: []ApiPokestopInvasionAvailable{}, - Lures: []ApiPokestopLureAvailable{}, - Showcases: []ApiPokestopShowcaseAvailable{}, + Invasions: readInvasions(now), + Lures: readLures(now), + Showcases: readShowcases(now), } - // Quests (rewards + title/target) come solely from the maintained conditions map — distinct+counted. - // ApiQuestConditionResult and ApiPokestopQuestAvailable share identical fields (name/order/type), so - // a direct conversion carries every field without restating them. for _, c := range GetAvailableQuestConditions() { res.Quests = append(res.Quests, ApiPokestopQuestAvailable(c)) } - for id, n := range a.lures { - res.Lures = append(res.Lures, ApiPokestopLureAvailable{LureId: id, Count: n}) - } - for k, n := range a.shows { - k.Count = n - res.Showcases = append(res.Showcases, k) - } - for k, n := range a.inv { - k.Count = n - res.Invasions = append(res.Invasions, k) - } + log.Infof("available-pokestops: %d quests, %d invasions, %d lures, %d showcases (maintained)", + len(res.Quests), len(res.Invasions), len(res.Lures), len(res.Showcases)) return res } - -// GetAvailablePokestops: one range over resident pokestops (lures, showcases, -// invasions + the quest-reward verification tally); quest options come from -// the maintained conditions map in result(). -func GetAvailablePokestops(now int64) *ApiAvailablePokestops { - start := time.Now() - acc := newPokestopAvailAcc() - fortLookupCache.Range(func(_ string, fl FortLookup) bool { - if fl.FortType == POKESTOP { - acc.ingest(&fl, now) - } - return true - }) - res := acc.result() - verifyQuestAggregate(acc.rewards) // alert if the maintained map drifted from the direct FortLookup tally - logAvailablePokestops(time.Since(start), acc.forts, acc.incidents, res) - return res -} - -// questRewardKey is the reward signature shared by the maintained conditions map (minus title/target) -// and the FortLookup reward tally used to detect reconciliation drift. -type questRewardKey struct { - WithAr bool - RewardType, ItemId, Amount, PokemonId, FormId int16 -} - -// verifyQuestAggregate is a Debug-level diagnostic, not a production alarm. It cross-checks the -// maintained conditions map against a direct FortLookup tally. Invariant: for each reward signature, -// sum(map counts over title/target) == resident forts carrying it. In practice benign, transient -// divergences occur routinely and are indistinguishable here from a real reconciliation bug: -// -// - Read-skew: the FortLookup reward tally and GetAvailableQuestConditions() are read at different -// instants, while updatePokestopLookup does its Store(FortLookup) then reconcile(map) non-atomically. -// - Pokestop→gym conversion lag: the maintained map keeps a converted stop's quest count until the -// stale pokestopCache entry evicts (up to the fort TTL, ~25h), but this range no longer tallies it. -// -// So a persistent mismatch is not reliably distinguishable from noise at this log level; a proper -// metric-based drift alarm (excluding converted stops from the comparison) is a documented follow-up. -func verifyQuestAggregate(fortRewards map[questRewardKey]int) { - mapRewards := map[questRewardKey]int{} - for _, c := range GetAvailableQuestConditions() { - mapRewards[questRewardKey{c.WithAr, c.RewardType, c.ItemId, c.Amount, c.PokemonId, c.FormId}] += c.Count - } - desync := 0 - for k, fortN := range fortRewards { - if mapRewards[k] != fortN { - desync++ - log.Debugf("quest aggregate desync %+v: fortLookup=%d map=%d", k, fortN, mapRewards[k]) - } - } - for k := range mapRewards { - if _, ok := fortRewards[k]; !ok { - desync++ - log.Debugf("quest aggregate desync %+v: fortLookup=0 map=%d", k, mapRewards[k]) - } - } - if desync > 0 { - log.Debugf("quest aggregate desync: %d reward signatures differ (FortLookup tally vs maintained map)", desync) - } -} - -// logAvailablePokestops records the available-pokestops build time in the -// api_scan_duration histogram (StatsCollector.ObserveApiScan) and logs a -// summary of the scan. -func logAvailablePokestops(dur time.Duration, forts, incidents int, res *ApiAvailablePokestops) { - if statsCollector != nil { - statsCollector.ObserveApiScan("available-pokestops", dur.Seconds()) - } - log.Infof("available-pokestops built in %s: scanned %d forts / %d incidents -> %d quests, %d invasions, %d lures, %d showcases", - dur, forts, incidents, len(res.Quests), len(res.Invasions), len(res.Lures), len(res.Showcases)) -} diff --git a/decoder/api_pokestop_available_test.go b/decoder/api_pokestop_available_test.go index bb2fa175..b5c7f750 100644 --- a/decoder/api_pokestop_available_test.go +++ b/decoder/api_pokestop_available_test.go @@ -1,30 +1,24 @@ package decoder -import ( - "testing" +import "testing" - "github.com/puzpuzpuz/xsync/v4" -) - -// TestGetAvailablePokestops seeds fortLookupCache + the quest-conditions map -// directly (initFortRtree pulls in pokestopCache/gymCache/stationCache wiring -// that isn't set up in this unit test, so we init only what this aggregate -// reads: fortLookupCache and the questConditionCount/questFortKeys pair via -// initQuestConditions). +// TestGetAvailablePokestops seeds the maintained lure/showcase/invasion maps +// via the observe hooks (not fortLookupCache — GetAvailablePokestops no +// longer scans it) plus the quest-conditions map via initQuestConditions. func TestGetAvailablePokestops(t *testing.T) { - fortLookupCache = xsync.NewMap[string, FortLookup]() + initFortAvailability() initQuestConditions() now := int64(1_000_000) // quest reward + condition via the maintained map (the sole quest source) adjustQuestConditions([]questConditionKey{{RewardType: 2, ItemId: 1, Title: "catch_x", Target: 3}}, +1) - // one fort: active lure, EXPIRED showcase (excluded), active grunt incident — all read in one range - fortLookupCache.Store("s1", FortLookup{ - FortType: POKESTOP, LureId: 501, LureExpireTimestamp: now + 100, + // one fort: active lure, EXPIRED showcase (excluded) + observePokestop(&FortLookup{ + LureId: 501, LureExpireTimestamp: now + 100, ContestPokemonId: 1, ShowcaseExpiry: now - 1, // expired -> excluded - Incidents: []FortLookupIncident{ - {DisplayType: 1, Character: 5, Confirmed: true, Slot1PokemonId: 41, ExpireTimestamp: now + 100}, - }, - }) + }, now) + // active grunt incident + observeInvasion(&FortLookupIncident{DisplayType: 1, Character: 5, Confirmed: true, Slot1PokemonId: 41, ExpireTimestamp: now + 100}, now) + res := GetAvailablePokestops(now) if len(res.Lures) != 1 || res.Lures[0].LureId != 501 { t.Fatalf("lure: %+v", res.Lures) diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 21e09e75..a53a21c0 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -219,6 +219,15 @@ func updatePokestopLookup(pokestop *Pokestop) { return nl, xsync.UpdateOp }) + observePokestop(&FortLookup{ + LureId: pokestop.LureId, + LureExpireTimestamp: pokestop.LureExpireTimestamp.ValueOrZero(), + ContestPokemonId: int16(pokestop.ShowcasePokemon.ValueOrZero()), + ContestPokemonForm: int16(pokestop.ShowcasePokemonForm.ValueOrZero()), + ContestPokemonType: int8(pokestop.ShowcasePokemonType.ValueOrZero()), + ShowcaseExpiry: pokestop.ShowcaseExpiry.ValueOrZero(), + }, time.Now().Unix()) + // This is the sole writer of a pokestop's FortLookup entry, so it is also // the single place quest-condition counts are reconciled: it fires on // cache-miss load, every save (incl. quest change), and startup preload. @@ -279,6 +288,7 @@ func updatePokestopIncidentLookup(pokestopId string, incident *Incident) { Slot1Form: int16(incident.Slot1Form.ValueOrZero()), ExpireTimestamp: incident.ExpirationTime, } + observeInvasion(&updated, now) // Atomic per-key read-modify-write via Compute — see updatePokestopLookup // for the cross-lock-domain clobber this prevents. fortLookupCache.Compute(pokestopId, func(existing FortLookup, loaded bool) (FortLookup, xsync.ComputeOp) { diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go index e5342cf9..a6ab918a 100644 --- a/decoder/fort_availability.go +++ b/decoder/fort_availability.go @@ -135,3 +135,71 @@ func readBattles(now int64) []ApiStationBattleAvailable { }) return out } + +// observePokestop records the lure and showcase options active on a pokestop. +// LureId 0 means no lure; ContestPokemonId 0 means no active showcase — both +// are gated here since observeExpiry only gates on expiry, not these sentinels. +func observePokestop(fl *FortLookup, now int64) { + if fl.LureId != 0 { + observeExpiry(lureExpiry, fl.LureId, fl.LureExpireTimestamp, now) + } + if fl.ContestPokemonId != 0 { + observeExpiry(showcaseExpiry, showcaseKey{fl.ContestPokemonId, fl.ContestPokemonForm, fl.ContestPokemonType}, fl.ShowcaseExpiry, now) + } +} + +// observeInvasion records the active invasion signature on one incident. +func observeInvasion(inc *FortLookupIncident, now int64) { + observeExpiry(invasionExpiry, invasionKey{ + Character: inc.Character, DisplayType: int16(inc.DisplayType), Confirmed: inc.Confirmed, + Slot1PokemonId: inc.Slot1PokemonId, Slot1Form: inc.Slot1Form, + }, inc.ExpireTimestamp, now) +} + +// readLures emits the distinct active lure ids, pruning expired keys. +// Strong Range (not RangeRelaxed): each key visited at most once. +func readLures(now int64) []ApiPokestopLureAvailable { + out := []ApiPokestopLureAvailable{} + lureExpiry.Range(func(k int16, exp int64) bool { + if exp > now { + out = append(out, ApiPokestopLureAvailable{LureId: k}) + } else { + pruneExpired(lureExpiry, k, now) + } + return true + }) + return out +} + +// readShowcases emits the distinct active showcase options, pruning expired +// keys. Strong Range (not RangeRelaxed): each key visited at most once. +func readShowcases(now int64) []ApiPokestopShowcaseAvailable { + out := []ApiPokestopShowcaseAvailable{} + showcaseExpiry.Range(func(k showcaseKey, exp int64) bool { + if exp > now { + out = append(out, ApiPokestopShowcaseAvailable{PokemonId: k.PokemonId, Form: k.Form, TypeId: k.TypeId}) + } else { + pruneExpired(showcaseExpiry, k, now) + } + return true + }) + return out +} + +// readInvasions emits the distinct active invasion signatures, pruning +// expired keys. Strong Range (not RangeRelaxed): each key visited at most once. +func readInvasions(now int64) []ApiPokestopInvasionAvailable { + out := []ApiPokestopInvasionAvailable{} + invasionExpiry.Range(func(k invasionKey, exp int64) bool { + if exp > now { + out = append(out, ApiPokestopInvasionAvailable{ + Character: k.Character, DisplayType: k.DisplayType, Confirmed: k.Confirmed, + Slot1PokemonId: k.Slot1PokemonId, Slot1Form: k.Slot1Form, + }) + } else { + pruneExpired(invasionExpiry, k, now) + } + return true + }) + return out +} diff --git a/decoder/fort_availability_test.go b/decoder/fort_availability_test.go index 2a6b7100..c9953478 100644 --- a/decoder/fort_availability_test.go +++ b/decoder/fort_availability_test.go @@ -71,3 +71,41 @@ func TestObserveStationBattlesAndRead(t *testing.T) { t.Fatal("all battles expired -> empty") } } + +func TestObservePokestopAggregatesAndRead(t *testing.T) { + initFortAvailability() + now := int64(1000) + + // lure + showcase on one stop + observePokestop(&FortLookup{ + LureId: 501, LureExpireTimestamp: 2000, + ContestPokemonId: 25, ContestPokemonForm: 0, ContestPokemonType: 0, ShowcaseExpiry: 2000, + }, now) + // expired lure + no showcase -> both ignored + observePokestop(&FortLookup{LureId: 502, LureExpireTimestamp: 500}, now) + + // invasions (per incident) + observeInvasion(&FortLookupIncident{Character: 5, DisplayType: 1, Confirmed: true, Slot1PokemonId: 41, ExpireTimestamp: 2000}, now) + observeInvasion(&FortLookupIncident{DisplayType: 9, ExpireTimestamp: 2000}, now) // showcase incident, character 0 + observeInvasion(&FortLookupIncident{Character: 30, DisplayType: 3, ExpireTimestamp: 500}, now) // expired + + if l := readLures(now); len(l) != 1 || l[0].LureId != 501 { + t.Fatalf("lures: %+v", l) + } + if s := readShowcases(now); len(s) != 1 || s[0].PokemonId != 25 { + t.Fatalf("showcases: %+v", s) + } + inv := readInvasions(now) + if len(inv) != 2 { + t.Fatalf("want 2 invasions, got %d: %+v", len(inv), inv) + } + for _, in := range inv { + if in.Character == 30 { + t.Fatal("expired invasion leaked") + } + } + // everything expires + if len(readLures(3000)) != 0 || len(readShowcases(3000)) != 0 || len(readInvasions(3000)) != 0 { + t.Fatal("all pokestop aggregates should expire to empty") + } +} From 69bdf40cff1aaaec986884ee08a2a15b1dd3ce64 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 21:51:28 +0100 Subject: [PATCH 19/29] feat(availability): combined endpoint reads maintained indexes; drop fort scan Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available.go | 12 +++------ decoder/api_fort_available_test.go | 40 ++++++++++-------------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go index 0f9cb4ca..b69cfaf8 100644 --- a/decoder/api_fort_available.go +++ b/decoder/api_fort_available.go @@ -1,14 +1,12 @@ package decoder import ( - "time" - log "github.com/sirupsen/logrus" ) // ApiAvailableForts is the whole-instance availability snapshot for every fort // type, served by GET /api/fort/available. Each section is read from its own -// maintained index (see decoder/fort_availability.go) — no fortLookupCache scan. +// maintained index (see decoder/fort_availability.go). type ApiAvailableForts struct { Pokestops *ApiAvailablePokestops `json:"pokestops" doc:"Pokestop availability (same shape as /api/pokestop/available)"` Gyms *ApiAvailableGyms `json:"gyms" doc:"Gym availability (same shape as /api/gym/available)"` @@ -18,15 +16,13 @@ type ApiAvailableForts struct { // GetAvailableForts assembles all three availability sections from the // maintained indexes — no fortLookupCache scan. func GetAvailableForts(now int64) *ApiAvailableForts { - start := time.Now() res := &ApiAvailableForts{ Pokestops: GetAvailablePokestops(now), Gyms: GetAvailableGyms(now), Stations: GetAvailableStations(now), } - if statsCollector != nil { - statsCollector.ObserveApiScan("available-forts", time.Since(start).Seconds()) - } - log.Infof("available-forts built in %s (pokestops/gyms/stations maintained)", time.Since(start)) + log.Infof("available-forts: %d raid, %d lure, %d invasion, %d showcase, %d battle options (maintained)", + len(res.Gyms.Raids), len(res.Pokestops.Lures), len(res.Pokestops.Invasions), + len(res.Pokestops.Showcases), len(res.Stations.Battles)) return res } diff --git a/decoder/api_fort_available_test.go b/decoder/api_fort_available_test.go index 908c126c..ff46b571 100644 --- a/decoder/api_fort_available_test.go +++ b/decoder/api_fort_available_test.go @@ -2,32 +2,22 @@ package decoder import ( "testing" - - "github.com/puzpuzpuz/xsync/v4" ) -// TestGetAvailableForts locks that the single-pass combined builder produces -// the same aggregates as the three per-type builders over the same cache. +// TestGetAvailableForts locks that the combined builder assembles all three +// availability sections from the maintained maps, matching the per-type +// builders over the same maps. func TestGetAvailableForts(t *testing.T) { - t.Skip("rebuilt in Task 4: combined reads maintained maps") - fortLookupCache = xsync.NewMap[string, FortLookup]() + initFortAvailability() initQuestConditions() now := int64(1_000_000) - fortLookupCache.Store("g1", FortLookup{ - FortType: GYM, TeamId: 1, AvailableSlots: 2, - RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: now + 100, - }) - fortLookupCache.Store("p1", FortLookup{ - FortType: POKESTOP, LureId: 501, LureExpireTimestamp: now + 100, - Incidents: []FortLookupIncident{{Character: 5, DisplayType: 1, ExpireTimestamp: now + 100}}, - }) - fortLookupCache.Store("s1", FortLookup{ - FortType: STATION, - StationBattles: []FortLookupStationBattle{ - {BattleLevel: 5, BattlePokemonId: 150, BattleEndTimestamp: now + 100}, - }, - }) + observeRaid(&FortLookup{RaidLevel: 5, RaidPokemonId: 150, RaidEndTimestamp: now + 100}, now) + observePokestop(&FortLookup{LureId: 501, LureExpireTimestamp: now + 100}, now) + observeInvasion(&FortLookupIncident{Character: 5, DisplayType: 1, ExpireTimestamp: now + 100}, now) + observeStationBattles(&FortLookup{StationBattles: []FortLookupStationBattle{ + {BattleLevel: 5, BattlePokemonId: 150, BattleEndTimestamp: now + 100}, + }}, now) combined := GetAvailableForts(now) if len(combined.Gyms.Raids) != 1 { @@ -40,13 +30,9 @@ func TestGetAvailableForts(t *testing.T) { t.Fatalf("stations: %+v", combined.Stations) } - // parity with the per-type builders over the same cache - perGym := GetAvailableGyms(now) - perStop := GetAvailablePokestops(now) - perStation := GetAvailableStations(now) - if len(perGym.Raids) != len(combined.Gyms.Raids) || - len(perStop.Lures) != len(combined.Pokestops.Lures) || - len(perStation.Battles) != len(combined.Stations.Battles) { + // parity: combined sections equal the per-type reads over the same maps + if len(GetAvailableGyms(now).Raids) != len(combined.Gyms.Raids) || + len(GetAvailableStations(now).Battles) != len(combined.Stations.Battles) { t.Fatal("combined diverges from per-type builders") } } From ed2f30fb90260519f3904034cde3928814c42f2d Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 22:10:53 +0100 Subject: [PATCH 20/29] fix(availability): one combined log line, accurate route docs, hook-wiring tests GetAvailableForts now builds its three sections from the internal build/read helpers instead of the public per-type GetAvailable* wrappers, so /api/fort/available emits one Info line instead of four. Reworded two stale route Descriptions (quest-only count, no fort scan). Added end-to-end hook tests that drive the real update* functions so a deleted observeX(...) call fails the suite. Co-Authored-By: Claude Fable 5 --- decoder/api_fort_available.go | 11 +- decoder/api_pokestop_available.go | 17 ++- decoder/fort_availability_hooks_test.go | 163 ++++++++++++++++++++++++ routes_huma.go | 4 +- 4 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 decoder/fort_availability_hooks_test.go diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go index b69cfaf8..90b28e4d 100644 --- a/decoder/api_fort_available.go +++ b/decoder/api_fort_available.go @@ -14,12 +14,15 @@ type ApiAvailableForts struct { } // GetAvailableForts assembles all three availability sections from the -// maintained indexes — no fortLookupCache scan. +// maintained indexes — no fortLookupCache scan. It calls the internal +// build/read helpers directly (not the public GetAvailable{Pokestops,Gyms, +// Stations} wrappers), so it emits exactly one combined Info line instead of +// one per section plus its own. func GetAvailableForts(now int64) *ApiAvailableForts { res := &ApiAvailableForts{ - Pokestops: GetAvailablePokestops(now), - Gyms: GetAvailableGyms(now), - Stations: GetAvailableStations(now), + Pokestops: buildAvailablePokestops(now), + Gyms: &ApiAvailableGyms{Raids: readRaids(now)}, + Stations: &ApiAvailableStations{Battles: readBattles(now)}, } log.Infof("available-forts: %d raid, %d lure, %d invasion, %d showcase, %d battle options (maintained)", len(res.Gyms.Raids), len(res.Pokestops.Lures), len(res.Pokestops.Invasions), diff --git a/decoder/api_pokestop_available.go b/decoder/api_pokestop_available.go index 0a94b76e..2c3cf40b 100644 --- a/decoder/api_pokestop_available.go +++ b/decoder/api_pokestop_available.go @@ -54,9 +54,13 @@ type ApiAvailablePokestops struct { Showcases []ApiPokestopShowcaseAvailable `json:"showcases" doc:"Distinct active showcase focus pokemon/type"` } -// GetAvailablePokestops reads the maintained lure/showcase/invasion indexes and -// the maintained quest-conditions aggregate (quests unchanged) — no fort scan. -func GetAvailablePokestops(now int64) *ApiAvailablePokestops { +// buildAvailablePokestops assembles the pokestop availability snapshot from +// the maintained lure/showcase/invasion indexes and the maintained +// quest-conditions aggregate (quests unchanged) — no fort scan, no logging. +// Shared by GetAvailablePokestops (which logs its own line for the per-type +// endpoint) and GetAvailableForts (which folds these counts into its single +// combined log line instead of logging again here). +func buildAvailablePokestops(now int64) *ApiAvailablePokestops { res := &ApiAvailablePokestops{ Quests: []ApiPokestopQuestAvailable{}, Invasions: readInvasions(now), @@ -66,6 +70,13 @@ func GetAvailablePokestops(now int64) *ApiAvailablePokestops { for _, c := range GetAvailableQuestConditions() { res.Quests = append(res.Quests, ApiPokestopQuestAvailable(c)) } + return res +} + +// GetAvailablePokestops reads the maintained lure/showcase/invasion indexes and +// the maintained quest-conditions aggregate (quests unchanged) — no fort scan. +func GetAvailablePokestops(now int64) *ApiAvailablePokestops { + res := buildAvailablePokestops(now) log.Infof("available-pokestops: %d quests, %d invasions, %d lures, %d showcases (maintained)", len(res.Quests), len(res.Invasions), len(res.Lures), len(res.Showcases)) return res diff --git a/decoder/fort_availability_hooks_test.go b/decoder/fort_availability_hooks_test.go new file mode 100644 index 00000000..623ae897 --- /dev/null +++ b/decoder/fort_availability_hooks_test.go @@ -0,0 +1,163 @@ +package decoder + +import ( + "testing" + "time" + + "github.com/guregu/null/v6" +) + +// These tests drive the REAL fort update functions (the ones wired into the +// save paths in fortRtree.go) end-to-end and assert the resulting option +// surfaces through the public GetAvailable* readers. Unlike +// fort_availability_test.go (which exercises the observe*/read* primitives +// directly), these exist to catch a deleted observeX(...) call inside an +// update function — a regression the primitive-level tests cannot see, +// since they never call the update functions at all. + +// TestUpdateGymLookupHookWiresRaidAvailability must fail if updateGymLookup's +// observeRaid(...) call is removed. +func TestUpdateGymLookupHookWiresRaidAvailability(t *testing.T) { + initFortAvailability() + now := time.Now().Unix() + + gym := &Gym{GymData: GymData{ + Id: "hook-gym-raid", + Lat: 1, + Lon: 2, + RaidLevel: null.IntFrom(5), + RaidPokemonId: null.IntFrom(150), + RaidPokemonForm: null.IntFrom(0), + RaidEndTimestamp: null.IntFrom(now + 1800), + }} + updateGymLookup(gym) + + got := GetAvailableGyms(now) + found := false + for _, r := range got.Raids { + if r.RaidLevel == 5 && r.PokemonId == 150 { + found = true + break + } + } + if !found { + t.Fatalf("expected raid option from updateGymLookup to surface via GetAvailableGyms, got %+v", got.Raids) + } +} + +// TestUpdateStationLookupWithBattlesHookWiresBattleAvailability must fail if +// updateStationLookupWithBattles's observeStationBattles(...) call is removed. +func TestUpdateStationLookupWithBattlesHookWiresBattleAvailability(t *testing.T) { + initFortAvailability() + now := time.Now().Unix() + + station := &Station{StationData: StationData{ + Id: "hook-station-battle", + Lat: 1, + Lon: 2, + StartTime: now - 3600, + EndTime: now + 3600, + Updated: now, + }} + battles := []StationBattleData{ + { + StationId: station.Id, + BattleLevel: 3, + BattleStart: now - 60, + BattleEnd: now + 1800, + BattlePokemonId: null.IntFrom(527), + }, + } + updateStationLookupWithBattles(station, battles) + + got := GetAvailableStations(now) + found := false + for _, b := range got.Battles { + if b.BattleLevel == 3 && b.PokemonId == 527 { + found = true + break + } + } + if !found { + t.Fatalf("expected battle option from updateStationLookupWithBattles to surface via GetAvailableStations, got %+v", got.Battles) + } +} + +// TestUpdatePokestopLookupHookWiresLureAndShowcaseAvailability must fail if +// updatePokestopLookup's observePokestop(...) call is removed. +func TestUpdatePokestopLookupHookWiresLureAndShowcaseAvailability(t *testing.T) { + initFortAvailability() + initQuestConditions() // updatePokestopLookup also reconciles quest conditions + now := time.Now().Unix() + + stop := &Pokestop{PokestopData: PokestopData{ + Id: "hook-stop-lure-showcase", + Lat: 1, + Lon: 2, + LureId: 501, + LureExpireTimestamp: null.IntFrom(now + 1800), + ShowcasePokemon: null.IntFrom(25), + ShowcasePokemonForm: null.IntFrom(0), + ShowcasePokemonType: null.IntFrom(0), + ShowcaseExpiry: null.IntFrom(now + 1800), + }} + updatePokestopLookup(stop) + + got := GetAvailablePokestops(now) + + foundLure := false + for _, l := range got.Lures { + if l.LureId == 501 { + foundLure = true + break + } + } + if !foundLure { + t.Fatalf("expected lure option from updatePokestopLookup to surface via GetAvailablePokestops, got %+v", got.Lures) + } + + foundShowcase := false + for _, s := range got.Showcases { + if s.PokemonId == 25 { + foundShowcase = true + break + } + } + if !foundShowcase { + t.Fatalf("expected showcase option from updatePokestopLookup to surface via GetAvailablePokestops, got %+v", got.Showcases) + } +} + +// TestUpdatePokestopIncidentLookupHookWiresInvasionAvailability must fail if +// updatePokestopIncidentLookup's observeInvasion(...) call is removed. The +// pokestop's FortLookup is seeded resident first (as fort_incident_id_test.go +// does), matching how a real incident save always follows a resident stop. +func TestUpdatePokestopIncidentLookupHookWiresInvasionAvailability(t *testing.T) { + initFortAvailability() + now := time.Now().Unix() + + const id = "hook-stop-invasion" + fortLookupCache.Store(id, FortLookup{FortType: POKESTOP, Lat: 1, Lon: 2}) + + inc := &Incident{IncidentData: IncidentData{ + Id: "hook-incident-1", + DisplayType: 1, + Character: 5, + Confirmed: true, + Slot1PokemonId: null.IntFrom(41), + ExpirationTime: now + 1800, + }} + updatePokestopIncidentLookup(id, inc) + + got := GetAvailablePokestops(now) + found := false + for _, iv := range got.Invasions { + if iv.Character == 5 && iv.Slot1PokemonId == 41 { + found = true + break + } + } + if !found { + t.Fatalf("expected invasion option from updatePokestopIncidentLookup to surface via GetAvailablePokestops, got %+v", got.Invasions) + } +} diff --git a/routes_huma.go b/routes_huma.go index e4bde12c..1adfe7bb 100644 --- a/routes_huma.go +++ b/routes_huma.go @@ -234,7 +234,7 @@ func registerFortScanRoutes(api huma.API) { Method: http.MethodGet, Path: "/api/pokestop/available", Summary: "List currently available pokestop rewards/invasions/lures/showcases", - Description: "Returns everything currently available on resident pokestops — distinct quest rewards (with title/target conditions), invasions, lures and showcases — from the in-memory fort cache, no DB scan. Whole-instance; requires fort_in_memory (503 otherwise). Presence-oriented: `count` is the number of resident forts offering each tuple; consumers typically use the distinct tuples to build filter options.", + Description: "Returns everything currently available on resident pokestops — distinct quest rewards (with title/target conditions), invasions, lures and showcases — from the in-memory fort cache, no DB scan. Whole-instance; requires fort_in_memory (503 otherwise). Presence-oriented: only quest entries carry `count` (the number of resident forts offering that exact reward+title+target); invasions/lures/showcases are distinct-option lists with no count. Consumers typically use the distinct tuples to build filter options.", Tags: []string{"Pokestop"}, Security: []map[string][]string{{securitySchemeName: {}}}, DefaultStatus: http.StatusOK, @@ -270,7 +270,7 @@ func registerFortScanRoutes(api huma.API) { Method: http.MethodGet, Path: "/api/fort/available", Summary: "List available options for all fort types in one pass", - Description: "Pokestop, gym, and station availability aggregates (same shapes as the per-type /available endpoints) built from a single in-memory cache pass — use this instead of three per-type calls when refreshing everything. Whole-instance; requires fort_in_memory (503 otherwise).", + Description: "Pokestop, gym, and station availability aggregates (same shapes as the per-type /available endpoints) assembled from the maintained availability indexes (no fort scan) — use this instead of three per-type calls when refreshing everything. Whole-instance; requires fort_in_memory (503 otherwise).", Tags: []string{"Fort"}, Security: []map[string][]string{{securitySchemeName: {}}}, DefaultStatus: http.StatusOK, From 398185177aa33f9563e72027d1ded28e0bca159c Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 22:11:59 +0100 Subject: [PATCH 21/29] docs: mark maintained availability spec/plan built --- .../specs/2026-07-17-maintained-fort-availability-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md b/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md index 3bcc20af..a08199f7 100644 --- a/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md +++ b/docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md @@ -1,7 +1,7 @@ # Maintained Fort Availability Index — Design Spec - **Date:** 2026-07-17 -- **Status:** Approved design → planning +- **Status:** Built (2026-07-17) — feat/fort-scan-map-data, all tasks + final review complete - **Repo:** Golbat only (`feat/fort-scan-map-data`, worktree `~/GolandProjects/Golbat-wt/pokestop-available-api`, PR #385) - **Author:** James Berry (with Claude) - **Extends:** the fort availability endpoints (`/api/{pokestop,gym,station}/available` + combined `/api/fort/available`). From b208fef38b89f6b2435d410a03fe360eaf288989 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 22:15:40 +0100 Subject: [PATCH 22/29] fix(availability): surface type-based showcases (pokemon id 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A type-based showcase (ContestPokemonType set, ContestPokemonId 0 — consumer key `h`) was gated out by the pokemon-id-only check, inherited from the old scan. Observe when either the pokemon id or the type is set. Co-Authored-By: Claude Fable 5 --- decoder/fort_availability.go | 4 +++- decoder/fort_availability_test.go | 23 +++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go index a6ab918a..95342c53 100644 --- a/decoder/fort_availability.go +++ b/decoder/fort_availability.go @@ -143,7 +143,9 @@ func observePokestop(fl *FortLookup, now int64) { if fl.LureId != 0 { observeExpiry(lureExpiry, fl.LureId, fl.LureExpireTimestamp, now) } - if fl.ContestPokemonId != 0 { + // A showcase is either pokemon-based (ContestPokemonId) or type-based + // (ContestPokemonType, pokemon id 0 -> consumer key `h`); surface both. + if fl.ContestPokemonId != 0 || fl.ContestPokemonType != 0 { observeExpiry(showcaseExpiry, showcaseKey{fl.ContestPokemonId, fl.ContestPokemonForm, fl.ContestPokemonType}, fl.ShowcaseExpiry, now) } } diff --git a/decoder/fort_availability_test.go b/decoder/fort_availability_test.go index c9953478..79168b62 100644 --- a/decoder/fort_availability_test.go +++ b/decoder/fort_availability_test.go @@ -76,12 +76,14 @@ func TestObservePokestopAggregatesAndRead(t *testing.T) { initFortAvailability() now := int64(1000) - // lure + showcase on one stop + // lure + pokemon-based showcase on one stop observePokestop(&FortLookup{ LureId: 501, LureExpireTimestamp: 2000, ContestPokemonId: 25, ContestPokemonForm: 0, ContestPokemonType: 0, ShowcaseExpiry: 2000, }, now) - // expired lure + no showcase -> both ignored + // type-based showcase (pokemon id 0, type set) -> must also surface + observePokestop(&FortLookup{ContestPokemonId: 0, ContestPokemonType: 12, ShowcaseExpiry: 2000}, now) + // expired lure + no showcase (all zero) -> both ignored observePokestop(&FortLookup{LureId: 502, LureExpireTimestamp: 500}, now) // invasions (per incident) @@ -92,8 +94,21 @@ func TestObservePokestopAggregatesAndRead(t *testing.T) { if l := readLures(now); len(l) != 1 || l[0].LureId != 501 { t.Fatalf("lures: %+v", l) } - if s := readShowcases(now); len(s) != 1 || s[0].PokemonId != 25 { - t.Fatalf("showcases: %+v", s) + if s := readShowcases(now); len(s) != 2 { + t.Fatalf("want 2 showcases (pokemon-based + type-based), got %d: %+v", len(s), s) + } else { + var pokemon, typeOnly bool + for _, sc := range s { + if sc.PokemonId == 25 { + pokemon = true + } + if sc.PokemonId == 0 && sc.TypeId == 12 { + typeOnly = true + } + } + if !pokemon || !typeOnly { + t.Fatalf("missing pokemon-based(25) or type-based(type 12) showcase: %+v", s) + } } inv := readInvasions(now) if len(inv) != 2 { From ab4ae6a3f3875134f28e60f0d97970be031a24fd Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 22:34:44 +0100 Subject: [PATCH 23/29] style(availability): convert key structs directly (staticcheck S1016) --- decoder/fort_availability.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go index 95342c53..edc063e5 100644 --- a/decoder/fort_availability.go +++ b/decoder/fort_availability.go @@ -92,7 +92,7 @@ func readRaids(now int64) []ApiGymRaidAvailable { out := []ApiGymRaidAvailable{} raidExpiry.Range(func(k raidKey, exp int64) bool { if exp > now { - out = append(out, ApiGymRaidAvailable{RaidLevel: k.RaidLevel, PokemonId: k.PokemonId, Form: k.Form}) + out = append(out, ApiGymRaidAvailable(k)) } else { pruneExpired(raidExpiry, k, now) } @@ -127,7 +127,7 @@ func readBattles(now int64) []ApiStationBattleAvailable { out := []ApiStationBattleAvailable{} battleExpiry.Range(func(k battleKey, exp int64) bool { if exp > now { - out = append(out, ApiStationBattleAvailable{BattleLevel: k.BattleLevel, PokemonId: k.PokemonId, Form: k.Form}) + out = append(out, ApiStationBattleAvailable(k)) } else { pruneExpired(battleExpiry, k, now) } @@ -179,7 +179,7 @@ func readShowcases(now int64) []ApiPokestopShowcaseAvailable { out := []ApiPokestopShowcaseAvailable{} showcaseExpiry.Range(func(k showcaseKey, exp int64) bool { if exp > now { - out = append(out, ApiPokestopShowcaseAvailable{PokemonId: k.PokemonId, Form: k.Form, TypeId: k.TypeId}) + out = append(out, ApiPokestopShowcaseAvailable(k)) } else { pruneExpired(showcaseExpiry, k, now) } @@ -194,10 +194,7 @@ func readInvasions(now int64) []ApiPokestopInvasionAvailable { out := []ApiPokestopInvasionAvailable{} invasionExpiry.Range(func(k invasionKey, exp int64) bool { if exp > now { - out = append(out, ApiPokestopInvasionAvailable{ - Character: k.Character, DisplayType: k.DisplayType, Confirmed: k.Confirmed, - Slot1PokemonId: k.Slot1PokemonId, Slot1Form: k.Slot1Form, - }) + out = append(out, ApiPokestopInvasionAvailable(k)) } else { pruneExpired(invasionExpiry, k, now) } From 2350a9b5db78dc13a07efe4552535df62140f69f Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 22:41:26 +0100 Subject: [PATCH 24/29] docs(claude): add build/test/lint commands + maintained-availability note --- CLAUDE.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 269251eb..07f7b2a2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,6 +4,12 @@ Golbat is a high-performance Go backend that receives raw protobuf data from Pokemon GO game clients, decodes it, maintains an in-memory cache of all game entities, persists changes to MySQL via write-behind queues, dispatches webhooks, and serves a REST/gRPC API for querying entities with spatial and attribute-based filters. +## Build, Test, Lint + +- **Build:** `go build -tags go_json ./...` — the `go_json` tag is required (selects the go-json codec). +- **Test:** `go test ./decoder/` — the `decoder` package holds most logic and its `_test.go` files; `go test .` runs the root-package huma route tests. Some tests need the go-json build behavior: `go test -tags go_json ./...`. +- **Lint:** `golangci-lint run` — CI enforces it (staticcheck included). Run it before pushing. Example: staticcheck `S1016` wants `ApiFoo(k)` conversion instead of a field-by-field literal when two structs have identical fields. + ## Project Layout ``` @@ -390,6 +396,15 @@ This avoids iterating all filters for every pokemon. The `FortCombinedScanEndpoint` scans all three fort types in one pass and splits results by type. +#### Fort Availability (maintained, not scanned) + +The `/api/{pokestop,gym,station}/available` and combined `/api/fort/available` endpoints answer "which distinct filter options are active on any resident fort right now?" (lures, showcases, invasions, raids, battles). These are served from **maintained max-expiry indexes** (`decoder/fort_availability.go`), NOT a `fortLookupCache.Range` scan: + +- Each fort update function (`updatePokestopLookup` → lures/showcases, `updatePokestopIncidentLookup` → invasions, `updateGymLookup` → raids, `updateStationLookupWithBattles` → battles) calls an `observe*` that records each active option's *latest* expiry in a small `xsync.Map[optionKey, maxExpiry]` (atomic keep-larger; already-expired observations ignored). These hooks fire during preload too, warming the maps at startup. +- Reads (`GetAvailable*`) use the strong `Map.Range` (not `RangeRelaxed`), emit keys with `exp > now`, and **prune-on-read conditionally** (`Compute` delete-if-still-`<= now`, never a blind `Delete`, so a concurrent refresh isn't dropped). +- **Quests are the exception**: a quest can be *retracted* mid-life (`RemoveQuestsWithinGeofence`, event swap) while its daily `quest_expiry` is still hours away — max-expiry is monotonic and can't retract — so quests use the `reconcile` count aggregate (`questConditionCount` / `reconcileFortQuestConditions`), never the index. `GetAvailablePokestops` sources quests from `GetAvailableQuestConditions()`. +- Accepted trade-off: no periodic reconcile sweep, so a replaced option (egg→hatch, grunt confirm) can over-report until its own expiry passes — bounded and self-healing. Design: `docs/superpowers/specs/2026-07-17-maintained-fort-availability-design.md`. + ## Geofence Matching Area attribution (stats, webhook filtering) matches points against Koji / From 3fc8d9ffedef12da01199272ec7f88868195afeb Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 23:17:11 +0100 Subject: [PATCH 25/29] chore(availability): include quest count in the combined log line --- decoder/api_fort_available.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/decoder/api_fort_available.go b/decoder/api_fort_available.go index 90b28e4d..4d2760f5 100644 --- a/decoder/api_fort_available.go +++ b/decoder/api_fort_available.go @@ -24,8 +24,8 @@ func GetAvailableForts(now int64) *ApiAvailableForts { Gyms: &ApiAvailableGyms{Raids: readRaids(now)}, Stations: &ApiAvailableStations{Battles: readBattles(now)}, } - log.Infof("available-forts: %d raid, %d lure, %d invasion, %d showcase, %d battle options (maintained)", - len(res.Gyms.Raids), len(res.Pokestops.Lures), len(res.Pokestops.Invasions), - len(res.Pokestops.Showcases), len(res.Stations.Battles)) + log.Infof("available-forts: %d quest, %d raid, %d lure, %d invasion, %d showcase, %d battle options (maintained)", + len(res.Pokestops.Quests), len(res.Gyms.Raids), len(res.Pokestops.Lures), + len(res.Pokestops.Invasions), len(res.Pokestops.Showcases), len(res.Stations.Battles)) return res } From 630c0658c10a37dfe4a7890d2a3979d77ff2efba Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 17 Jul 2026 23:35:26 +0100 Subject: [PATCH 26/29] refactor(fort): dedup scan-result loops and availability readers Extract collectGymResults/collectPokestopResults/collectStationResults, shared by the single-type scan endpoints and FortCombinedScanEndpoint (the pokestop lock-order dance was previously copy-pasted in both). Collapse the five near-identical read* availability functions into a generic readAvailable[K,V] + one-line wrappers, centralising the prune-on-read and strong-Range invariants in one place. Behaviour-preserving; build/test/lint clean, availability tests pass -race. Co-Authored-By: Claude Opus 4.8 (1M context) --- decoder/api_fort.go | 137 ++++++++++++++++------------------- decoder/fort_availability.go | 83 ++++++--------------- 2 files changed, 84 insertions(+), 136 deletions(-) diff --git a/decoder/api_fort.go b/decoder/api_fort.go index 0689f38f..063edd61 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -356,13 +356,13 @@ func internalGetForts(fortType FortType, retrieveParameters ApiFortScan) ([]stri return returnKeys, fortsExamined, fortsSkipped, fortTreeCopy.Len() } -func GymScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiGymScanResult { - returnKeys, examined, skipped, total := internalGetForts(GYM, retrieveParameters) - results := make([]*ApiGymResult, 0, len(returnKeys)) - start := time.Now() - - for _, key := range returnKeys { - gym, unlock, err := GetGymRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanGym") +// collectGymResults loads each key's gym record read-only and builds its API +// result, always releasing the per-record lock. Shared by the single-type and +// combined scan endpoints; traceName distinguishes the caller in lock traces. +func collectGymResults(dbDetails db.DbDetails, keys []string, traceName string) []*ApiGymResult { + results := make([]*ApiGymResult, 0, len(keys)) + for _, key := range keys { + gym, unlock, err := GetGymRecordReadOnly(context.Background(), dbDetails, key, traceName) if err == nil && gym != nil { gymCopy := buildGymResult(gym) results = append(results, &gymCopy) @@ -371,31 +371,43 @@ func GymScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *Ap unlock() } } - log.Infof("GymScan - result buffer time %s, %d added", time.Since(start), len(results)) + return results +} - return &ApiGymScanResult{ - Gyms: results, - Examined: examined, - Skipped: skipped, - Total: total, +// collectStationResults loads each key's station record read-only and builds +// its API result, always releasing the per-record lock. Shared by the +// single-type and combined scan endpoints; traceName distinguishes the caller. +func collectStationResults(dbDetails db.DbDetails, keys []string, traceName string) []*ApiStationResult { + results := make([]*ApiStationResult, 0, len(keys)) + for _, key := range keys { + station, unlock, err := GetStationRecordReadOnly(context.Background(), dbDetails, key, traceName) + if err == nil && station != nil { + stationCopy := BuildStationResult(station) + results = append(results, &stationCopy) + } + if unlock != nil { + unlock() + } } + return results } -func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiPokestopScanResult { - returnKeys, examined, skipped, total := internalGetForts(POKESTOP, retrieveParameters) - results := make([]*ApiPokestopResult, 0, len(returnKeys)) - start := time.Now() - now := time.Now().Unix() - - for _, key := range returnKeys { - pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanpokemon") +// collectPokestopResults loads each key's pokestop record read-only and builds +// its API result. It releases the pokestop lock BEFORE collecting incidents to +// preserve lock-order (pokestop then incidents), then optionally attaches +// invasions. Shared by the single-type and combined scan endpoints; traceName +// distinguishes the caller in lock traces. +func collectPokestopResults(dbDetails db.DbDetails, keys []string, withIncidents bool, now int64, traceName string) []*ApiPokestopResult { + results := make([]*ApiPokestopResult, 0, len(keys)) + for _, key := range keys { + pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key, traceName) if err == nil && pokestop != nil { pokestopCopy := buildPokestopResult(pokestop) if unlock != nil { unlock() // release pokestop lock BEFORE locking incidents (lock-order) unlock = nil } - if retrieveParameters.WithIncidents { + if withIncidents { pokestopCopy.Invasions = CollectPokestopIncidents(context.Background(), dbDetails, key, now) } results = append(results, &pokestopCopy) @@ -404,6 +416,29 @@ func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails unlock() } } + return results +} + +func GymScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiGymScanResult { + returnKeys, examined, skipped, total := internalGetForts(GYM, retrieveParameters) + start := time.Now() + + results := collectGymResults(dbDetails, returnKeys, "API.GetScanGym") + log.Infof("GymScan - result buffer time %s, %d added", time.Since(start), len(results)) + + return &ApiGymScanResult{ + Gyms: results, + Examined: examined, + Skipped: skipped, + Total: total, + } +} + +func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiPokestopScanResult { + returnKeys, examined, skipped, total := internalGetForts(POKESTOP, retrieveParameters) + start := time.Now() + + results := collectPokestopResults(dbDetails, returnKeys, retrieveParameters.WithIncidents, time.Now().Unix(), "API.GetScanpokemon") log.Infof("PokestopScan - result buffer time %s, %d added", time.Since(start), len(results)) return &ApiPokestopScanResult{ @@ -416,19 +451,9 @@ func PokestopScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails func StationScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) *ApiStationScanResult { returnKeys, examined, skipped, total := internalGetForts(STATION, retrieveParameters) - results := make([]*ApiStationResult, 0, len(returnKeys)) start := time.Now() - for _, key := range returnKeys { - station, unlock, err := GetStationRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanStation") - if err == nil && station != nil { - stationCopy := BuildStationResult(station) - results = append(results, &stationCopy) - } - if unlock != nil { - unlock() - } - } + results := collectStationResults(dbDetails, returnKeys, "API.GetScanStation") log.Infof("StationScan - result buffer time %s, %d added", time.Since(start), len(results)) return &ApiStationScanResult{ @@ -442,50 +467,10 @@ func StationScanEndpoint(retrieveParameters ApiFortScan, dbDetails db.DbDetails) func FortCombinedScanEndpoint(retrieveParameters ApiFortCombinedScan, dbDetails db.DbDetails) *ApiFortCombinedScanResult { gymKeys, pokestopKeys, stationKeys, examined, skipped, total := internalGetFortsCombined(retrieveParameters) start := time.Now() - now := time.Now().Unix() - - gyms := make([]*ApiGymResult, 0, len(gymKeys)) - for _, key := range gymKeys { - gym, unlock, err := GetGymRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanGymPokemon") - if err == nil && gym != nil { - gymCopy := buildGymResult(gym) - gyms = append(gyms, &gymCopy) - } - if unlock != nil { - unlock() - } - } - - pokestops := make([]*ApiPokestopResult, 0, len(pokestopKeys)) - for _, key := range pokestopKeys { - pokestop, unlock, err := getPokestopRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanpokemonPokemon") - if err == nil && pokestop != nil { - pokestopCopy := buildPokestopResult(pokestop) - if unlock != nil { - unlock() // release pokestop lock BEFORE locking incidents (lock-order) - unlock = nil - } - if retrieveParameters.WithIncidents { - pokestopCopy.Invasions = CollectPokestopIncidents(context.Background(), dbDetails, key, now) - } - pokestops = append(pokestops, &pokestopCopy) - } - if unlock != nil { - unlock() - } - } - stations := make([]*ApiStationResult, 0, len(stationKeys)) - for _, key := range stationKeys { - station, unlock, err := GetStationRecordReadOnly(context.Background(), dbDetails, key, "API.GetScanStationPokemon") - if err == nil && station != nil { - stationCopy := BuildStationResult(station) - stations = append(stations, &stationCopy) - } - if unlock != nil { - unlock() - } - } + gyms := collectGymResults(dbDetails, gymKeys, "API.GetScanGymPokemon") + pokestops := collectPokestopResults(dbDetails, pokestopKeys, retrieveParameters.WithIncidents, time.Now().Unix(), "API.GetScanpokemonPokemon") + stations := collectStationResults(dbDetails, stationKeys, "API.GetScanStationPokemon") log.Infof("FortCombinedScan - result buffer time %s, %d+%d+%d added", time.Since(start), len(gyms), len(pokestops), len(stations)) diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go index edc063e5..f0aacc15 100644 --- a/decoder/fort_availability.go +++ b/decoder/fort_availability.go @@ -80,25 +80,32 @@ func pruneExpired[K comparable](m *xsync.Map[K, int64], key K, now int64) { }) } +// readAvailable emits the distinct still-active options from a maintained +// index, converting each live key to its API shape and pruning expired keys as +// it goes. It uses the strong Map.Range (each key visited at most once — +// RangeRelaxed could emit a duplicate); prune-on-read is the conditional +// pruneExpired, never a blind Delete. Returns a non-nil empty slice, never nil. +func readAvailable[K comparable, V any](m *xsync.Map[K, int64], now int64, conv func(K) V) []V { + out := []V{} + m.Range(func(k K, exp int64) bool { + if exp > now { + out = append(out, conv(k)) + } else { + pruneExpired(m, k, now) + } + return true + }) + return out +} + func observeRaid(fl *FortLookup, now int64) { if fl.RaidLevel > 0 { observeExpiry(raidExpiry, raidKey{fl.RaidLevel, fl.RaidPokemonId, fl.RaidPokemonForm}, fl.RaidEndTimestamp, now) } } -// readRaids emits the distinct active raid options, pruning expired keys. -// Strong Range (not RangeRelaxed): each key visited at most once. func readRaids(now int64) []ApiGymRaidAvailable { - out := []ApiGymRaidAvailable{} - raidExpiry.Range(func(k raidKey, exp int64) bool { - if exp > now { - out = append(out, ApiGymRaidAvailable(k)) - } else { - pruneExpired(raidExpiry, k, now) - } - return true - }) - return out + return readAvailable(raidExpiry, now, func(k raidKey) ApiGymRaidAvailable { return ApiGymRaidAvailable(k) }) } // observeStationBattles records every distinct active battle option on a @@ -121,19 +128,8 @@ func observeStationBattles(fl *FortLookup, now int64) { } } -// readBattles emits the distinct active station battle options, pruning -// expired keys. Strong Range (not RangeRelaxed): each key visited at most once. func readBattles(now int64) []ApiStationBattleAvailable { - out := []ApiStationBattleAvailable{} - battleExpiry.Range(func(k battleKey, exp int64) bool { - if exp > now { - out = append(out, ApiStationBattleAvailable(k)) - } else { - pruneExpired(battleExpiry, k, now) - } - return true - }) - return out + return readAvailable(battleExpiry, now, func(k battleKey) ApiStationBattleAvailable { return ApiStationBattleAvailable(k) }) } // observePokestop records the lure and showcase options active on a pokestop. @@ -158,47 +154,14 @@ func observeInvasion(inc *FortLookupIncident, now int64) { }, inc.ExpireTimestamp, now) } -// readLures emits the distinct active lure ids, pruning expired keys. -// Strong Range (not RangeRelaxed): each key visited at most once. func readLures(now int64) []ApiPokestopLureAvailable { - out := []ApiPokestopLureAvailable{} - lureExpiry.Range(func(k int16, exp int64) bool { - if exp > now { - out = append(out, ApiPokestopLureAvailable{LureId: k}) - } else { - pruneExpired(lureExpiry, k, now) - } - return true - }) - return out + return readAvailable(lureExpiry, now, func(k int16) ApiPokestopLureAvailable { return ApiPokestopLureAvailable{LureId: k} }) } -// readShowcases emits the distinct active showcase options, pruning expired -// keys. Strong Range (not RangeRelaxed): each key visited at most once. func readShowcases(now int64) []ApiPokestopShowcaseAvailable { - out := []ApiPokestopShowcaseAvailable{} - showcaseExpiry.Range(func(k showcaseKey, exp int64) bool { - if exp > now { - out = append(out, ApiPokestopShowcaseAvailable(k)) - } else { - pruneExpired(showcaseExpiry, k, now) - } - return true - }) - return out + return readAvailable(showcaseExpiry, now, func(k showcaseKey) ApiPokestopShowcaseAvailable { return ApiPokestopShowcaseAvailable(k) }) } -// readInvasions emits the distinct active invasion signatures, pruning -// expired keys. Strong Range (not RangeRelaxed): each key visited at most once. func readInvasions(now int64) []ApiPokestopInvasionAvailable { - out := []ApiPokestopInvasionAvailable{} - invasionExpiry.Range(func(k invasionKey, exp int64) bool { - if exp > now { - out = append(out, ApiPokestopInvasionAvailable(k)) - } else { - pruneExpired(invasionExpiry, k, now) - } - return true - }) - return out + return readAvailable(invasionExpiry, now, func(k invasionKey) ApiPokestopInvasionAvailable { return ApiPokestopInvasionAvailable(k) }) } From a3c82e5f6116056dd6c2e019ce794e8c58f7cb43 Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 20 Jul 2026 11:56:45 +0100 Subject: [PATCH 27/29] feat(fort): dedicated max_fort_results scan cap, separate from pokemon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The in-memory fort scan (/api/{gym,pokestop,station}/scan and /api/fort/scan) capped its traversal at Tuning.MaxPokemonResults — so the fort map-data API shared the pokemon result limit and couldn't be tuned independently. Add a dedicated Tuning.MaxFortResults (koanf max_fort_results, default 9000) and use it for both fort scan caps. The per-request Limit still min()s it, so ReactMap keeps sending its queryLimits value; operators can now raise the fort ceiling without touching max_pokemon_results. Co-Authored-By: Claude Opus 4.8 (1M context) --- config.toml.example | 1 + config/config.go | 1 + config/reader.go | 1 + decoder/api_fort.go | 4 ++-- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config.toml.example b/config.toml.example index 6647256f..e26bf4b5 100644 --- a/config.toml.example +++ b/config.toml.example @@ -76,6 +76,7 @@ url = "http://localhost:4201" [tuning] max_pokemon_distance = 100 # Maximum distance in kilometers for searching pokemon max_pokemon_results = 3000 # Maximum number of pokemon to return +max_fort_results = 9000 # Maximum number of forts (gyms/pokestops/stations) to return per in-memory scan; separate from max_pokemon_results so the fort map-data API can be tuned independently extended_timeout = false # Extend timeouts for processing, hopefully not needed write_behind_startup_delay = 120 # Writes will be queued for this quiet time write_behind_worker_count = 16 # Maximum number of parallel batch writes. diff --git a/config/config.go b/config/config.go index 09ef6954..1f3d7dae 100644 --- a/config/config.go +++ b/config/config.go @@ -133,6 +133,7 @@ type database struct { type tuning struct { ExtendedTimeout bool `koanf:"extended_timeout"` MaxPokemonResults int `koanf:"max_pokemon_results"` + MaxFortResults int `koanf:"max_fort_results"` MaxPokemonDistance float64 `koanf:"max_pokemon_distance"` ProfileRoutes bool `koanf:"profile_routes"` ProfileContention bool `koanf:"profile_contention"` // Enable mutex/block profiling (has overhead) diff --git a/config/reader.go b/config/reader.go index eec3ff4c..72b8bc1c 100644 --- a/config/reader.go +++ b/config/reader.go @@ -55,6 +55,7 @@ func ReadConfig() (configDefinition, error) { }, Tuning: tuning{ MaxPokemonResults: 3000, + MaxFortResults: 9000, MaxPokemonDistance: 100, MaxConcurrentProactiveIVSwitch: 6, ReduceUpdates: false, diff --git a/decoder/api_fort.go b/decoder/api_fort.go index 063edd61..8f4486c5 100644 --- a/decoder/api_fort.go +++ b/decoder/api_fort.go @@ -295,7 +295,7 @@ func internalGetForts(fortType FortType, retrieveParameters ApiFortScan) ([]stri minLocation := retrieveParameters.Min.Location() maxLocation := retrieveParameters.Max.Location() - maxForts := config.Config.Tuning.MaxPokemonResults + maxForts := config.Config.Tuning.MaxFortResults if retrieveParameters.Limit > 0 && retrieveParameters.Limit < maxForts { maxForts = retrieveParameters.Limit } @@ -491,7 +491,7 @@ func internalGetFortsCombined(retrieveParameters ApiFortCombinedScan) (gymKeys, minLocation := retrieveParameters.Min.Location() maxLocation := retrieveParameters.Max.Location() - maxForts := config.Config.Tuning.MaxPokemonResults + maxForts := config.Config.Tuning.MaxFortResults if retrieveParameters.Limit > 0 && retrieveParameters.Limit < maxForts { maxForts = retrieveParameters.Limit } From e8c4505328c43c6e5ad2c0efc8d0c29765af86c2 Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 20 Jul 2026 12:10:07 +0100 Subject: [PATCH 28/29] feat(fort): expose confirmed invasion slots 2 and 3 in availability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The invasion availability aggregate exposed only slot 1, so a consumer could only advertise confirmed rocket-reward filters for the lead pokemon; SQL adds configured rewards from slots 1/2/3. Thread slots 2/3 through FortLookupIncident, invasionKey, observeInvasion and ApiPokestopInvasionAvailable (the incident data already carries them). Index cardinality is unchanged: a confirmed grunt's lineup is deterministic per character, so (character, slot1) already determines slots 2/3 — the key is only a few bytes wider, no new distinct entries. Supports ReactMap Mygod review comment #9. Co-Authored-By: Claude Opus 4.8 (1M context) --- decoder/api_pokestop_available.go | 4 ++++ decoder/fortRtree.go | 4 ++++ decoder/fort_availability.go | 6 ++++++ decoder/fort_availability_test.go | 9 +++++++-- decoder/station_battle.go | 4 ++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/decoder/api_pokestop_available.go b/decoder/api_pokestop_available.go index 2c3cf40b..48d00487 100644 --- a/decoder/api_pokestop_available.go +++ b/decoder/api_pokestop_available.go @@ -29,6 +29,10 @@ type ApiPokestopInvasionAvailable struct { Confirmed bool `json:"confirmed" doc:"True when the lineup is confirmed (grunts only)"` Slot1PokemonId int16 `json:"slot1_pokemon_id" doc:"Confirmed lead pokemon id (grunts only), else 0"` Slot1Form int16 `json:"slot1_form" doc:"Confirmed lead pokemon form, else 0"` + Slot2PokemonId int16 `json:"slot2_pokemon_id" doc:"Confirmed slot-2 pokemon id (grunts only), else 0"` + Slot2Form int16 `json:"slot2_form" doc:"Confirmed slot-2 pokemon form, else 0"` + Slot3PokemonId int16 `json:"slot3_pokemon_id" doc:"Confirmed slot-3 pokemon id (grunts only), else 0"` + Slot3Form int16 `json:"slot3_form" doc:"Confirmed slot-3 pokemon form, else 0"` } // ApiPokestopLureAvailable is one distinct active lure type currently carried diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index a53a21c0..96a9523d 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -286,6 +286,10 @@ func updatePokestopIncidentLookup(pokestopId string, incident *Incident) { Confirmed: incident.Confirmed, Slot1PokemonId: int16(incident.Slot1PokemonId.ValueOrZero()), Slot1Form: int16(incident.Slot1Form.ValueOrZero()), + Slot2PokemonId: int16(incident.Slot2PokemonId.ValueOrZero()), + Slot2Form: int16(incident.Slot2Form.ValueOrZero()), + Slot3PokemonId: int16(incident.Slot3PokemonId.ValueOrZero()), + Slot3Form: int16(incident.Slot3Form.ValueOrZero()), ExpireTimestamp: incident.ExpirationTime, } observeInvasion(&updated, now) diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go index f0aacc15..c81c266a 100644 --- a/decoder/fort_availability.go +++ b/decoder/fort_availability.go @@ -23,6 +23,10 @@ type invasionKey struct { Confirmed bool Slot1PokemonId int16 Slot1Form int16 + Slot2PokemonId int16 + Slot2Form int16 + Slot3PokemonId int16 + Slot3Form int16 } type raidKey struct { @@ -151,6 +155,8 @@ func observeInvasion(inc *FortLookupIncident, now int64) { observeExpiry(invasionExpiry, invasionKey{ Character: inc.Character, DisplayType: int16(inc.DisplayType), Confirmed: inc.Confirmed, Slot1PokemonId: inc.Slot1PokemonId, Slot1Form: inc.Slot1Form, + Slot2PokemonId: inc.Slot2PokemonId, Slot2Form: inc.Slot2Form, + Slot3PokemonId: inc.Slot3PokemonId, Slot3Form: inc.Slot3Form, }, inc.ExpireTimestamp, now) } diff --git a/decoder/fort_availability_test.go b/decoder/fort_availability_test.go index 79168b62..3b4dc0e6 100644 --- a/decoder/fort_availability_test.go +++ b/decoder/fort_availability_test.go @@ -86,8 +86,8 @@ func TestObservePokestopAggregatesAndRead(t *testing.T) { // expired lure + no showcase (all zero) -> both ignored observePokestop(&FortLookup{LureId: 502, LureExpireTimestamp: 500}, now) - // invasions (per incident) - observeInvasion(&FortLookupIncident{Character: 5, DisplayType: 1, Confirmed: true, Slot1PokemonId: 41, ExpireTimestamp: 2000}, now) + // invasions (per incident) — confirmed lineup carries all three slots + observeInvasion(&FortLookupIncident{Character: 5, DisplayType: 1, Confirmed: true, Slot1PokemonId: 41, Slot2PokemonId: 42, Slot2Form: 1, Slot3PokemonId: 43, ExpireTimestamp: 2000}, now) observeInvasion(&FortLookupIncident{DisplayType: 9, ExpireTimestamp: 2000}, now) // showcase incident, character 0 observeInvasion(&FortLookupIncident{Character: 30, DisplayType: 3, ExpireTimestamp: 500}, now) // expired @@ -118,6 +118,11 @@ func TestObservePokestopAggregatesAndRead(t *testing.T) { if in.Character == 30 { t.Fatal("expired invasion leaked") } + if in.Character == 5 { + if in.Slot2PokemonId != 42 || in.Slot2Form != 1 || in.Slot3PokemonId != 43 { + t.Fatalf("confirmed invasion lost slots 2/3: %+v", in) + } + } } // everything expires if len(readLures(3000)) != 0 || len(readShowcases(3000)) != 0 || len(readInvasions(3000)) != 0 { diff --git a/decoder/station_battle.go b/decoder/station_battle.go index 23225dab..d583bdaf 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -55,6 +55,10 @@ type FortLookupIncident struct { Confirmed bool Slot1PokemonId int16 Slot1Form int16 + Slot2PokemonId int16 + Slot2Form int16 + Slot3PokemonId int16 + Slot3Form int16 ExpireTimestamp int64 // used to skip expired incidents at filter time } From 959cb170d9858401fa8d5a0e08018e8d63a8859e Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 27 Jul 2026 22:44:22 +0100 Subject: [PATCH 29/29] feat(fort): availability emits DB-null bosses; expose per-battle updated Two consumer-driven refinements to the maintained fort availability output. A. Emit null (not the internal 0 sentinel) where the source DB column is NULL. updateFort*Lookup collapses null.Int columns to int16(0) via ValueOrZero when populating the FortLookup index; the availability read conv now reverses that so a consumer sees the same value it would read from the DB row: - station battle boss, raid boss (egg), showcase focus, and confirmed invasion slots expose *int16 pokemon_id/form (null when absent). - showcase type_id is *int8 (null for a pokemon-based showcase). - a pokemon id 0 is never legitimate (pokedex ids start at 1), so it maps to null and its paired form nulls WITH it; form 0 alongside a present id stays 0 (a valid form). Character/display_type are genuine int16 columns and are unchanged. Quests are unchanged: their per-reward-type fields never leak a 0 into a consumer key. B. ApiStationBattleResult now carries `updated` (from the stored StationBattleData.Updated), so multi-battle consumers get the real per-battle timestamp instead of the station-wide fallback. Co-Authored-By: Claude Opus 4.8 (1M context) --- decoder/api_gym_available.go | 8 ++-- decoder/api_gym_available_test.go | 4 +- decoder/api_pokestop_available.go | 24 +++++------ decoder/api_station.go | 1 + decoder/api_station_available.go | 6 +-- decoder/api_station_available_test.go | 2 +- decoder/fort_availability.go | 55 +++++++++++++++++++++++-- decoder/fort_availability_hooks_test.go | 8 ++-- decoder/fort_availability_test.go | 14 ++++--- decoder/station_battle.go | 1 + 10 files changed, 88 insertions(+), 35 deletions(-) diff --git a/decoder/api_gym_available.go b/decoder/api_gym_available.go index a7fdc7f3..1708342e 100644 --- a/decoder/api_gym_available.go +++ b/decoder/api_gym_available.go @@ -5,11 +5,11 @@ import ( ) // ApiGymRaidAvailable is one distinct active raid option on resident gyms. -// PokemonId 0 means an egg (no boss yet). ReactMap derives its e/r/boss keys. +// A null PokemonId means an egg (no boss yet). ReactMap derives its e/r/boss keys. type ApiGymRaidAvailable struct { - RaidLevel int8 `json:"raid_level" doc:"Raid level/tier"` - PokemonId int16 `json:"pokemon_id" doc:"Raid boss pokemon id; 0 = egg (unhatched)"` - Form int16 `json:"form" doc:"Raid boss form id, else 0"` + RaidLevel int8 `json:"raid_level" doc:"Raid level/tier"` + PokemonId *int16 `json:"pokemon_id" doc:"Raid boss pokemon id; null for an unhatched egg"` + Form *int16 `json:"form" doc:"Raid boss form id (0 is a valid form); null for an unhatched egg"` } // ApiAvailableGyms is the whole-instance gym filter snapshot. Only raids are diff --git a/decoder/api_gym_available_test.go b/decoder/api_gym_available_test.go index 240abf3a..60be8f26 100644 --- a/decoder/api_gym_available_test.go +++ b/decoder/api_gym_available_test.go @@ -16,10 +16,10 @@ func TestGetAvailableGyms(t *testing.T) { var bosses, eggs int for _, r := range res.Raids { - if r.PokemonId == 999 { + if r.PokemonId != nil && *r.PokemonId == 999 { t.Fatalf("expired raid leaked: %+v", r) } - if r.PokemonId == 0 { + if r.PokemonId == nil { // egg: unhatched boss is null, not 0 eggs++ } else { bosses++ diff --git a/decoder/api_pokestop_available.go b/decoder/api_pokestop_available.go index 48d00487..1bd984e5 100644 --- a/decoder/api_pokestop_available.go +++ b/decoder/api_pokestop_available.go @@ -24,15 +24,15 @@ type ApiPokestopQuestAvailable struct { // showcase character + slot1 reward) currently present on resident // pokestops, with how many forts carry it. Sourced from FortLookup.Incidents. type ApiPokestopInvasionAvailable struct { - Character int16 `json:"character" doc:"Invasion character id (grunt/leader/giovanni); 0 for non-rocket displays"` - DisplayType int16 `json:"display_type" doc:"Incident display type (1-4 rocket, 7 goldstop, 8 kecleon, 9 showcase/contest)"` - Confirmed bool `json:"confirmed" doc:"True when the lineup is confirmed (grunts only)"` - Slot1PokemonId int16 `json:"slot1_pokemon_id" doc:"Confirmed lead pokemon id (grunts only), else 0"` - Slot1Form int16 `json:"slot1_form" doc:"Confirmed lead pokemon form, else 0"` - Slot2PokemonId int16 `json:"slot2_pokemon_id" doc:"Confirmed slot-2 pokemon id (grunts only), else 0"` - Slot2Form int16 `json:"slot2_form" doc:"Confirmed slot-2 pokemon form, else 0"` - Slot3PokemonId int16 `json:"slot3_pokemon_id" doc:"Confirmed slot-3 pokemon id (grunts only), else 0"` - Slot3Form int16 `json:"slot3_form" doc:"Confirmed slot-3 pokemon form, else 0"` + Character int16 `json:"character" doc:"Invasion character id (grunt/leader/giovanni); 0 for non-rocket displays"` + DisplayType int16 `json:"display_type" doc:"Incident display type (1-4 rocket, 7 goldstop, 8 kecleon, 9 showcase/contest)"` + Confirmed bool `json:"confirmed" doc:"True when the lineup is confirmed (grunts only)"` + Slot1PokemonId *int16 `json:"slot1_pokemon_id" doc:"Confirmed lead pokemon id (grunts only); null when unconfirmed/absent"` + Slot1Form *int16 `json:"slot1_form" doc:"Confirmed lead pokemon form (0 is a valid form); null when unconfirmed/absent"` + Slot2PokemonId *int16 `json:"slot2_pokemon_id" doc:"Confirmed slot-2 pokemon id (grunts only); null when unconfirmed/absent"` + Slot2Form *int16 `json:"slot2_form" doc:"Confirmed slot-2 pokemon form (0 is a valid form); null when unconfirmed/absent"` + Slot3PokemonId *int16 `json:"slot3_pokemon_id" doc:"Confirmed slot-3 pokemon id (grunts only); null when unconfirmed/absent"` + Slot3Form *int16 `json:"slot3_form" doc:"Confirmed slot-3 pokemon form (0 is a valid form); null when unconfirmed/absent"` } // ApiPokestopLureAvailable is one distinct active lure type currently carried @@ -44,9 +44,9 @@ type ApiPokestopLureAvailable struct { // ApiPokestopShowcaseAvailable is one distinct active showcase contest // (pokemon/form/type) currently run by resident pokestops. type ApiPokestopShowcaseAvailable struct { - PokemonId int16 `json:"pokemon_id" doc:"Showcase focus pokemon id, else 0"` - Form int16 `json:"form" doc:"Showcase focus pokemon form, else 0"` - TypeId int8 `json:"type_id" doc:"Showcase focus pokemon type id (type-based showcases), else 0"` + PokemonId *int16 `json:"pokemon_id" doc:"Showcase focus pokemon id; null for a type-based showcase"` + Form *int16 `json:"form" doc:"Showcase focus pokemon form (0 is a valid form); null for a type-based showcase"` + TypeId *int8 `json:"type_id" doc:"Showcase focus pokemon type id (type-based showcases); null for a pokemon-based showcase"` } // ApiAvailablePokestops is the whole-instance snapshot served by diff --git a/decoder/api_station.go b/decoder/api_station.go index 41496a02..6f9fe6fc 100644 --- a/decoder/api_station.go +++ b/decoder/api_station.go @@ -44,6 +44,7 @@ type ApiStationBattleResult struct { BattleLevel int16 `json:"battle_level" doc:"Battle level"` BattleStart int64 `json:"battle_start" doc:"Unix timestamp when the battle starts"` BattleEnd int64 `json:"battle_end" doc:"Unix timestamp when the battle ends"` + Updated int64 `json:"updated" doc:"Unix timestamp when this battle record was last updated"` BattlePokemonId *int64 `json:"battle_pokemon_id" doc:"Pokedex ID of the battle pokemon"` BattlePokemonForm *int64 `json:"battle_pokemon_form" doc:"Form ID of the battle pokemon"` BattlePokemonCostume *int64 `json:"battle_pokemon_costume" doc:"Costume ID of the battle pokemon"` diff --git a/decoder/api_station_available.go b/decoder/api_station_available.go index d6dfef94..9a43a63d 100644 --- a/decoder/api_station_available.go +++ b/decoder/api_station_available.go @@ -7,9 +7,9 @@ import ( // ApiStationBattleAvailable is one distinct active (battle_level, pokemon, form) // option on resident stations. ReactMap derives its - and j keys. type ApiStationBattleAvailable struct { - BattleLevel int8 `json:"battle_level" doc:"Max battle level"` - PokemonId int16 `json:"pokemon_id" doc:"Battle pokemon id, else 0"` - Form int16 `json:"form" doc:"Battle pokemon form id, else 0"` + BattleLevel int8 `json:"battle_level" doc:"Max battle level"` + PokemonId *int16 `json:"pokemon_id" doc:"Battle pokemon id; null when the boss is not yet known"` + Form *int16 `json:"form" doc:"Battle pokemon form id (0 is a valid form); null when the boss is not yet known"` } // ApiAvailableStations is the whole-instance station filter snapshot served by diff --git a/decoder/api_station_available_test.go b/decoder/api_station_available_test.go index 949e6efb..ebf2539b 100644 --- a/decoder/api_station_available_test.go +++ b/decoder/api_station_available_test.go @@ -30,7 +30,7 @@ func TestGetAvailableStations(t *testing.T) { t.Fatalf("battles: %+v", res.Battles) } for _, b := range res.Battles { - if b.BattleLevel == 0 || b.PokemonId == 1 { + if b.BattleLevel == 0 || (b.PokemonId != nil && *b.PokemonId == 1) { t.Fatalf("excluded battle leaked: %+v", b) } } diff --git a/decoder/fort_availability.go b/decoder/fort_availability.go index c81c266a..ba199c50 100644 --- a/decoder/fort_availability.go +++ b/decoder/fort_availability.go @@ -102,6 +102,20 @@ func readAvailable[K comparable, V any](m *xsync.Map[K, int64], now int64, conv return out } +// nullablePokemonForm reverses the null.Int -> int16(0) collapse that +// updateFort*Lookup applies when populating a FortLookup (see fortRtree.go's +// ValueOrZero calls). Availability output emits the DB-visible value: a 0 +// pokemon id means the source column was NULL (unknown boss / egg / no reward), +// so both the id AND its form return null. A pokemon id is never legitimately 0 +// (pokedex ids start at 1). Form 0, in contrast, is a VALID form and is +// preserved whenever the id is present — never null a form on form==0 alone. +func nullablePokemonForm(id, form int16) (*int16, *int16) { + if id == 0 { + return nil, nil + } + return &id, &form +} + func observeRaid(fl *FortLookup, now int64) { if fl.RaidLevel > 0 { observeExpiry(raidExpiry, raidKey{fl.RaidLevel, fl.RaidPokemonId, fl.RaidPokemonForm}, fl.RaidEndTimestamp, now) @@ -109,7 +123,10 @@ func observeRaid(fl *FortLookup, now int64) { } func readRaids(now int64) []ApiGymRaidAvailable { - return readAvailable(raidExpiry, now, func(k raidKey) ApiGymRaidAvailable { return ApiGymRaidAvailable(k) }) + return readAvailable(raidExpiry, now, func(k raidKey) ApiGymRaidAvailable { + id, form := nullablePokemonForm(k.PokemonId, k.Form) + return ApiGymRaidAvailable{RaidLevel: k.RaidLevel, PokemonId: id, Form: form} + }) } // observeStationBattles records every distinct active battle option on a @@ -133,7 +150,10 @@ func observeStationBattles(fl *FortLookup, now int64) { } func readBattles(now int64) []ApiStationBattleAvailable { - return readAvailable(battleExpiry, now, func(k battleKey) ApiStationBattleAvailable { return ApiStationBattleAvailable(k) }) + return readAvailable(battleExpiry, now, func(k battleKey) ApiStationBattleAvailable { + id, form := nullablePokemonForm(k.PokemonId, k.Form) + return ApiStationBattleAvailable{BattleLevel: k.BattleLevel, PokemonId: id, Form: form} + }) } // observePokestop records the lure and showcase options active on a pokestop. @@ -165,9 +185,36 @@ func readLures(now int64) []ApiPokestopLureAvailable { } func readShowcases(now int64) []ApiPokestopShowcaseAvailable { - return readAvailable(showcaseExpiry, now, func(k showcaseKey) ApiPokestopShowcaseAvailable { return ApiPokestopShowcaseAvailable(k) }) + return readAvailable(showcaseExpiry, now, func(k showcaseKey) ApiPokestopShowcaseAvailable { + // A showcase is exactly one of pokemon-based or type-based (observePokestop + // gates on ContestPokemonId != 0 || ContestPokemonType != 0). Mirror the DB: + // pokemon-based -> pokemon/form set, type null; type-based -> pokemon/form + // null, type set. + id, form := nullablePokemonForm(k.PokemonId, k.Form) + var typeId *int8 + if k.TypeId != 0 { + t := k.TypeId + typeId = &t + } + return ApiPokestopShowcaseAvailable{PokemonId: id, Form: form, TypeId: typeId} + }) } func readInvasions(now int64) []ApiPokestopInvasionAvailable { - return readAvailable(invasionExpiry, now, func(k invasionKey) ApiPokestopInvasionAvailable { return ApiPokestopInvasionAvailable(k) }) + return readAvailable(invasionExpiry, now, func(k invasionKey) ApiPokestopInvasionAvailable { + // Character/DisplayType are genuine int16 columns (never a null collapse), + // so they pass through as-is; only the confirmed slot pokemon/form pairs + // reverse the ValueOrZero collapse. + s1id, s1form := nullablePokemonForm(k.Slot1PokemonId, k.Slot1Form) + s2id, s2form := nullablePokemonForm(k.Slot2PokemonId, k.Slot2Form) + s3id, s3form := nullablePokemonForm(k.Slot3PokemonId, k.Slot3Form) + return ApiPokestopInvasionAvailable{ + Character: k.Character, + DisplayType: k.DisplayType, + Confirmed: k.Confirmed, + Slot1PokemonId: s1id, Slot1Form: s1form, + Slot2PokemonId: s2id, Slot2Form: s2form, + Slot3PokemonId: s3id, Slot3Form: s3form, + } + }) } diff --git a/decoder/fort_availability_hooks_test.go b/decoder/fort_availability_hooks_test.go index 623ae897..9e16962f 100644 --- a/decoder/fort_availability_hooks_test.go +++ b/decoder/fort_availability_hooks_test.go @@ -35,7 +35,7 @@ func TestUpdateGymLookupHookWiresRaidAvailability(t *testing.T) { got := GetAvailableGyms(now) found := false for _, r := range got.Raids { - if r.RaidLevel == 5 && r.PokemonId == 150 { + if r.RaidLevel == 5 && r.PokemonId != nil && *r.PokemonId == 150 { found = true break } @@ -73,7 +73,7 @@ func TestUpdateStationLookupWithBattlesHookWiresBattleAvailability(t *testing.T) got := GetAvailableStations(now) found := false for _, b := range got.Battles { - if b.BattleLevel == 3 && b.PokemonId == 527 { + if b.BattleLevel == 3 && b.PokemonId != nil && *b.PokemonId == 527 { found = true break } @@ -118,7 +118,7 @@ func TestUpdatePokestopLookupHookWiresLureAndShowcaseAvailability(t *testing.T) foundShowcase := false for _, s := range got.Showcases { - if s.PokemonId == 25 { + if s.PokemonId != nil && *s.PokemonId == 25 { foundShowcase = true break } @@ -152,7 +152,7 @@ func TestUpdatePokestopIncidentLookupHookWiresInvasionAvailability(t *testing.T) got := GetAvailablePokestops(now) found := false for _, iv := range got.Invasions { - if iv.Character == 5 && iv.Slot1PokemonId == 41 { + if iv.Character == 5 && iv.Slot1PokemonId != nil && *iv.Slot1PokemonId == 41 { found = true break } diff --git a/decoder/fort_availability_test.go b/decoder/fort_availability_test.go index 3b4dc0e6..0d022565 100644 --- a/decoder/fort_availability_test.go +++ b/decoder/fort_availability_test.go @@ -19,7 +19,7 @@ func TestObserveExpiryAndReadRaids(t *testing.T) { t.Fatalf("want 2 raid options, got %d: %+v", len(got), got) } for _, r := range got { - if r.PokemonId == 999 { + if r.PokemonId != nil && *r.PokemonId == 999 { t.Fatal("expired raid must not appear") } } @@ -63,7 +63,7 @@ func TestObserveStationBattlesAndRead(t *testing.T) { t.Fatalf("want 3 battle options, got %d: %+v", len(got), got) } for _, b := range got { - if b.PokemonId == 999 { + if b.PokemonId != nil && *b.PokemonId == 999 { t.Fatal("expired battle leaked") } } @@ -99,10 +99,11 @@ func TestObservePokestopAggregatesAndRead(t *testing.T) { } else { var pokemon, typeOnly bool for _, sc := range s { - if sc.PokemonId == 25 { + if sc.PokemonId != nil && *sc.PokemonId == 25 { pokemon = true } - if sc.PokemonId == 0 && sc.TypeId == 12 { + // type-based: pokemon/form null, type set + if sc.PokemonId == nil && sc.TypeId != nil && *sc.TypeId == 12 { typeOnly = true } } @@ -119,7 +120,10 @@ func TestObservePokestopAggregatesAndRead(t *testing.T) { t.Fatal("expired invasion leaked") } if in.Character == 5 { - if in.Slot2PokemonId != 42 || in.Slot2Form != 1 || in.Slot3PokemonId != 43 { + // slots present -> non-null; form 0 stays 0, form 1 stays 1 + if in.Slot2PokemonId == nil || *in.Slot2PokemonId != 42 || + in.Slot2Form == nil || *in.Slot2Form != 1 || + in.Slot3PokemonId == nil || *in.Slot3PokemonId != 43 { t.Fatalf("confirmed invasion lost slots 2/3: %+v", in) } } diff --git a/decoder/station_battle.go b/decoder/station_battle.go index d583bdaf..e0f2d1ca 100644 --- a/decoder/station_battle.go +++ b/decoder/station_battle.go @@ -482,6 +482,7 @@ func buildApiStationBattleResults(battles []StationBattleData) []ApiStationBattl BattleLevel: battle.BattleLevel, BattleStart: battle.BattleStart, BattleEnd: battle.BattleEnd, + Updated: battle.Updated, BattlePokemonId: battle.BattlePokemonId.Ptr(), BattlePokemonForm: battle.BattlePokemonForm.Ptr(), BattlePokemonCostume: battle.BattlePokemonCostume.Ptr(),