Skip to content

perf: engine-independent decode wins (DiscardUnknown, HTTP buffer pool, PGO tooling) #389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jfberry merged 1 commit into from
Jul 29, 2026
Merged

perf: engine-independent decode wins (DiscardUnknown, HTTP buffer pool, PGO tooling) #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,43 @@ FLAGS = go_json
golbat: FORCE
go build -tags $(FLAGS) golbat

# --- Profile-guided optimization --------------------------------------------
# Go (>= 1.21) automatically applies ./default.pgo when building the golbat
# main package — locally and in the Docker build (which copies the repo). So
# committing default.pgo makes every build profile-guided; no flag needed.
# Refresh it occasionally (game updates shift hot paths), then commit:
#
# make pgo-capture # reads port + api_secret from config.toml
# GOLBAT_URL=https://host:9001 GOLBAT_SECRET=... make pgo-capture # or override
#
# Port and secret are auto-detected from config.toml so a local operator can
# just `make pgo-capture`; any of GOLBAT_URL / GOLBAT_HOST / GOLBAT_PORT /
# GOLBAT_SECRET / CONFIG override the detection.
CONFIG ?= config.toml
DETECTED_PORT := $(shell grep -E '^[[:space:]]*port[[:space:]]*=' $(CONFIG) 2>/dev/null | head -1 | sed -E 's/^[^=]*=[[:space:]]*([0-9]+).*/\1/')
DETECTED_SECRET := $(shell grep -E '^[[:space:]]*api_secret[[:space:]]*=' $(CONFIG) 2>/dev/null | head -1 | sed -E 's/^[^=]*=[[:space:]]*"([^"]*)".*/\1/')
GOLBAT_HOST ?= 127.0.0.1
GOLBAT_PORT ?= $(if $(DETECTED_PORT),$(DETECTED_PORT),9001)
GOLBAT_SECRET ?= $(DETECTED_SECRET)
GOLBAT_URL ?= http://$(GOLBAT_HOST):$(GOLBAT_PORT)
PGO_SECONDS ?= 120

pgo-config: FORCE ## show the URL/secret pgo-capture will use (from config.toml)
@echo "config file : $(CONFIG)"
@echo "capture URL : $(GOLBAT_URL)"
@echo "secret : $(if $(GOLBAT_SECRET),(set, $(shell printf %s '$(GOLBAT_SECRET)' | wc -c | tr -d ' ') chars),(none))"

pgo-capture: FORCE ## capture a CPU profile from a running golbat into default.pgo
curl -fsS $(if $(GOLBAT_SECRET),-H "X-Golbat-Secret: $(GOLBAT_SECRET)") \
"$(GOLBAT_URL)/debug/pprof/profile?seconds=$(PGO_SECONDS)" -o default.pgo.tmp
mv default.pgo.tmp default.pgo
@echo "captured default.pgo ($$(du -h default.pgo | cut -f1)) from $(GOLBAT_URL) — commit it to apply to all builds"

pgo-status: FORCE ## report whether builds will be profile-guided
@if [ -f default.pgo ]; then \
echo "default.pgo present ($$(du -h default.pgo | cut -f1)) — builds are profile-guided"; \
else \
echo "no default.pgo — builds are NOT profile-guided (run make pgo-capture against prod)"; \
fi

FORCE: ;
69 changes: 42 additions & 27 deletions decode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,21 @@ import (
"google.golang.org/protobuf/proto"
)

// clientProtoUnmarshalOpts decodes inbound game-client protos with
// DiscardUnknown. This is safe — and a measured win (+~3.7% decode rate,
// −5.5% allocated objects) — because Golbat only ever READS these protos and
// never re-serializes them, so protobuf-go's default behavior of retaining
// unknown fields (present because production payloads are a newer game version
// than our vbase schema) is pure allocation churn. Applies on both ingest
// paths, since both funnel their per-packet decode through the calls below.
var clientProtoUnmarshalOpts = proto.UnmarshalOptions{DiscardUnknown: true}

// unmarshalClientProto is proto.Unmarshal with DiscardUnknown. Use it for
// every inbound client proto (see clientProtoUnmarshalOpts).
func unmarshalClientProto(b []byte, m proto.Message) error {
return clientProtoUnmarshalOpts.Unmarshal(b, m)
}

func decode(ctx context.Context, method int, protoData *ProtoData) {
getMethodName := func(method int, trimString bool) string {
if val, ok := pogo.Method_name[int32(method)]; ok {
Expand Down Expand Up @@ -144,7 +159,7 @@ func decodeQuest(ctx context.Context, sDec []byte, haveAr *bool) string {
return "No AR quest info"
}
decodedQuest := &pogo.FortSearchOutProto{}
if err := proto.Unmarshal(sDec, decodedQuest); err != nil {
if err := unmarshalClientProto(sDec, decodedQuest); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeQuest("error", "parse")
return "Parse failure"
Expand All @@ -164,15 +179,15 @@ func decodeQuest(ctx context.Context, sDec []byte, haveAr *bool) string {
func decodeSocialActionWithRequest(request []byte, payload []byte) string {
var proxyRequestProto pogo.ProxyRequestProto

if err := proto.Unmarshal(request, &proxyRequestProto); err != nil {
if err := unmarshalClientProto(request, &proxyRequestProto); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeSocialActionWithRequest("error", "request_parse")
return fmt.Sprintf("Failed to parse %s", err)
}

var proxyResponseProto pogo.ProxyResponseProto

if err := proto.Unmarshal(payload, &proxyResponseProto); err != nil {
if err := unmarshalClientProto(payload, &proxyResponseProto); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeSocialActionWithRequest("error", "response_parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -199,7 +214,7 @@ func decodeSocialActionWithRequest(request []byte, payload []byte) string {

func decodeGetFriendDetails(payload []byte) string {
var getFriendDetailsOutProto pogo.InternalGetFriendDetailsOutProto
getFriendDetailsError := proto.Unmarshal(payload, &getFriendDetailsOutProto)
getFriendDetailsError := unmarshalClientProto(payload, &getFriendDetailsOutProto)

if getFriendDetailsError != nil {
statsCollector.IncDecodeGetFriendDetails("error", "parse")
Expand Down Expand Up @@ -229,7 +244,7 @@ func decodeGetFriendDetails(payload []byte) string {

func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byte) string {
var searchPlayerOutProto pogo.InternalSearchPlayerOutProto
searchPlayerOutError := proto.Unmarshal(payload, &searchPlayerOutProto)
searchPlayerOutError := unmarshalClientProto(payload, &searchPlayerOutProto)

if searchPlayerOutError != nil {
log.Errorf("Failed to parse %s", searchPlayerOutError)
Expand All @@ -243,7 +258,7 @@ func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byt
}

var searchPlayerProto pogo.InternalSearchPlayerProto
searchPlayerError := proto.Unmarshal(proxyRequestProto.GetPayload(), &searchPlayerProto)
searchPlayerError := unmarshalClientProto(proxyRequestProto.GetPayload(), &searchPlayerProto)

if searchPlayerError != nil || searchPlayerProto.GetFriendCode() == "" {
statsCollector.IncDecodeSearchPlayer("error", "parse")
Expand All @@ -263,7 +278,7 @@ func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byt

func decodeFortDetails(ctx context.Context, sDec []byte) string {
decodedFort := &pogo.FortDetailsOutProto{}
if err := proto.Unmarshal(sDec, decodedFort); err != nil {
if err := unmarshalClientProto(sDec, decodedFort); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeFortDetails("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -284,7 +299,7 @@ func decodeFortDetails(ctx context.Context, sDec []byte) string {

func decodeGetMapForts(ctx context.Context, sDec []byte) string {
decodedMapForts := &pogo.GetMapFortsOutProto{}
if err := proto.Unmarshal(sDec, decodedMapForts); err != nil {
if err := unmarshalClientProto(sDec, decodedMapForts); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeGetMapForts("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand Down Expand Up @@ -317,7 +332,7 @@ func decodeGetMapForts(ctx context.Context, sDec []byte) string {

func decodeGetRoutes(ctx context.Context, payload []byte) string {
getRoutesOutProto := &pogo.GetRoutesOutProto{}
if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil {
if err := unmarshalClientProto(payload, getRoutesOutProto); err != nil {
return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err)
}

Expand Down Expand Up @@ -358,7 +373,7 @@ func decodeGetRoutes(ctx context.Context, payload []byte) string {

func decodeGetGymInfo(ctx context.Context, sDec []byte) string {
decodedGymInfo := &pogo.GymGetInfoOutProto{}
if err := proto.Unmarshal(sDec, decodedGymInfo); err != nil {
if err := unmarshalClientProto(sDec, decodedGymInfo); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeGetGymInfo("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -377,7 +392,7 @@ func decodeGetGymInfo(ctx context.Context, sDec []byte) string {

func decodeEncounter(ctx context.Context, sDec []byte, username string, timestampMs int64) string {
decodedEncounterInfo := &pogo.EncounterOutProto{}
if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil {
if err := unmarshalClientProto(sDec, decodedEncounterInfo); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeEncounter("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -396,7 +411,7 @@ func decodeEncounter(ctx context.Context, sDec []byte, username string, timestam

func decodeDiskEncounter(ctx context.Context, sDec []byte, username string) string {
decodedEncounterInfo := &pogo.DiskEncounterOutProto{}
if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil {
if err := unmarshalClientProto(sDec, decodedEncounterInfo); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeDiskEncounter("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -415,7 +430,7 @@ func decodeDiskEncounter(ctx context.Context, sDec []byte, username string) stri

func decodeStartIncident(ctx context.Context, sDec []byte) string {
decodedIncident := &pogo.StartIncidentOutProto{}
if err := proto.Unmarshal(sDec, decodedIncident); err != nil {
if err := unmarshalClientProto(sDec, decodedIncident); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeStartIncident("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -435,7 +450,7 @@ func decodeStartIncident(ctx context.Context, sDec []byte) string {
func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) string {
decodeOpenInvasionRequest := &pogo.OpenInvasionCombatSessionProto{}

if err := proto.Unmarshal(request, decodeOpenInvasionRequest); err != nil {
if err := unmarshalClientProto(request, decodeOpenInvasionRequest); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeOpenInvasion("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -445,7 +460,7 @@ func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) str
}

decodedOpenInvasionResponse := &pogo.OpenInvasionCombatSessionOutProto{}
if err := proto.Unmarshal(payload, decodedOpenInvasionResponse); err != nil {
if err := unmarshalClientProto(payload, decodedOpenInvasionResponse); err != nil {
log.Errorf("Failed to parse %s", err)
statsCollector.IncDecodeOpenInvasion("error", "parse")
return fmt.Sprintf("Failed to parse %s", err)
Expand All @@ -465,7 +480,7 @@ func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) str
func decodeGMO(ctx context.Context, protoData *ProtoData, scanParameters decoder.ScanParameters) string {
decodedGmo := &pogo.GetMapObjectsOutProto{}

if err := proto.Unmarshal(protoData.Data, decodedGmo); err != nil {
if err := unmarshalClientProto(protoData.Data, decodedGmo); err != nil {
statsCollector.IncDecodeGMO("error", "parse")
log.Errorf("Failed to parse %s", err)
}
Expand Down Expand Up @@ -587,14 +602,14 @@ func isCellNotEmpty(mapCell *pogo.ClientMapCellProto) bool {

func decodeGetContestData(ctx context.Context, request []byte, data []byte) string {
var decodedContestData pogo.GetContestDataOutProto
if err := proto.Unmarshal(data, &decodedContestData); err != nil {
if err := unmarshalClientProto(data, &decodedContestData); err != nil {
log.Errorf("Failed to parse GetContestDataOutProto %s", err)
return fmt.Sprintf("Failed to parse GetContestDataOutProto %s", err)
}

var decodedContestDataRequest pogo.GetContestDataProto
if request != nil {
if err := proto.Unmarshal(request, &decodedContestDataRequest); err != nil {
if err := unmarshalClientProto(request, &decodedContestDataRequest); err != nil {
log.Errorf("Failed to parse GetContestDataProto %s", err)
return fmt.Sprintf("Failed to parse GetContestDataProto %s", err)
}
Expand All @@ -604,7 +619,7 @@ func decodeGetContestData(ctx context.Context, request []byte, data []byte) stri

func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data []byte) string {
var decodedPokemonSizeContestEntry pogo.GetPokemonSizeLeaderboardEntryOutProto
if err := proto.Unmarshal(data, &decodedPokemonSizeContestEntry); err != nil {
if err := unmarshalClientProto(data, &decodedPokemonSizeContestEntry); err != nil {
log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err)
return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err)
}
Expand All @@ -615,7 +630,7 @@ func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data

var decodedPokemonSizeContestEntryRequest pogo.GetPokemonSizeLeaderboardEntryProto
if request != nil {
if err := proto.Unmarshal(request, &decodedPokemonSizeContestEntryRequest); err != nil {
if err := unmarshalClientProto(request, &decodedPokemonSizeContestEntryRequest); err != nil {
log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err)
return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err)
}
Expand All @@ -626,14 +641,14 @@ func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data

func decodeGetStationDetails(ctx context.Context, request []byte, data []byte) string {
var decodedGetStationDetails pogo.GetStationedPokemonDetailsOutProto
if err := proto.Unmarshal(data, &decodedGetStationDetails); err != nil {
if err := unmarshalClientProto(data, &decodedGetStationDetails); err != nil {
log.Errorf("Failed to parse GetStationedPokemonDetailsOutProto %s", err)
return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsOutProto %s", err)
}

var decodedGetStationDetailsRequest pogo.GetStationedPokemonDetailsProto
if request != nil {
if err := proto.Unmarshal(request, &decodedGetStationDetailsRequest); err != nil {
if err := unmarshalClientProto(request, &decodedGetStationDetailsRequest); err != nil {
log.Errorf("Failed to parse GetStationedPokemonDetailsProto %s", err)
return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsProto %s", err)
}
Expand All @@ -651,14 +666,14 @@ func decodeGetStationDetails(ctx context.Context, request []byte, data []byte) s

func decodeTappable(ctx context.Context, request, data []byte, username string, timestampMs int64) string {
var tappable pogo.ProcessTappableOutProto
if err := proto.Unmarshal(data, &tappable); err != nil {
if err := unmarshalClientProto(data, &tappable); err != nil {
log.Errorf("Failed to parse %s", err)
return fmt.Sprintf("Failed to parse ProcessTappableOutProto %s", err)
}

var tappableRequest pogo.ProcessTappableProto
if request != nil {
if err := proto.Unmarshal(request, &tappableRequest); err != nil {
if err := unmarshalClientProto(request, &tappableRequest); err != nil {
log.Errorf("Failed to parse %s", err)
return fmt.Sprintf("Failed to parse ProcessTappableProto %s", err)
}
Expand All @@ -676,14 +691,14 @@ func decodeTappable(ctx context.Context, request, data []byte, username string,

func decodeGetEventRsvp(ctx context.Context, request []byte, data []byte) string {
var rsvp pogo.GetEventRsvpsOutProto
if err := proto.Unmarshal(data, &rsvp); err != nil {
if err := unmarshalClientProto(data, &rsvp); err != nil {
log.Errorf("Failed to parse %s", err)
return fmt.Sprintf("Failed to parse GetEventRsvpsOutProto %s", err)
}

var rsvpRequest pogo.GetEventRsvpsProto
if request != nil {
if err := proto.Unmarshal(request, &rsvpRequest); err != nil {
if err := unmarshalClientProto(request, &rsvpRequest); err != nil {
log.Errorf("Failed to parse %s", err)
return fmt.Sprintf("Failed to parse GetEventRsvpsProto %s", err)
}
Expand All @@ -705,7 +720,7 @@ func decodeGetEventRsvp(ctx context.Context, request []byte, data []byte) string

func decodeGetEventRsvpCount(ctx context.Context, data []byte) string {
var rsvp pogo.GetEventRsvpCountOutProto
if err := proto.Unmarshal(data, &rsvp); err != nil {
if err := unmarshalClientProto(data, &rsvp); err != nil {
log.Errorf("Failed to parse %s", err)
return fmt.Sprintf("Failed to parse GetEventRsvpCountOutProto %s", err)
}
Expand Down
3 changes: 1 addition & 2 deletions decode_nebula.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"golbat/pogo"

log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
)

// decodeNebula routes on the typed context (the proto `oneof context` case,
Expand Down Expand Up @@ -36,7 +35,7 @@ func decodeNebula(ctx context.Context, endpoint string, nd *NebulaData) string {

func decodeNebulaInvasionState(ctx context.Context, fortId, incidentId string, payload []byte) string {
var out pogo.BattleStateOutProto
if err := proto.Unmarshal(payload, &out); err != nil {
if err := unmarshalClientProto(payload, &out); err != nil {
return "failed to parse BattleStateOutProto"
}
return decoder.UpdateIncidentLineupFromBattleState(ctx, dbDetails, fortId, incidentId, &out)
Expand Down
3 changes: 1 addition & 2 deletions decode_push_gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"golbat/pogo"

log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
)

// decodePushGateway classifies a push-gateway message by message_type, unmarshals
Expand All @@ -22,7 +21,7 @@ func decodePushGateway(ctx context.Context, messageType string, payload []byte)
}

var msg pogo.PushGatewayMessage
if err := proto.Unmarshal(payload, &msg); err != nil {
if err := unmarshalClientProto(payload, &msg); err != nil {
log.Warnf("PushGateway: failed to parse %s: %v", messageType, err)
return
}
Expand Down
Loading