Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions harnesses/solana-exec/cmd/collector/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,20 +45,28 @@ func collect(ctx context.Context, db *store.DB, h *helius.Client, plt, feeAccoun
}

const sigLimit = 100
// Fetch signatures newer than last seen. Results are newest-first.
sigs, err := h.GetSignaturesForAddress(ctx, feeAccount, sigLimit, cursor.LastSig)
if err != nil {
return fmt.Errorf("get sigs: %w", err)
// Paginate through ALL signatures newer than cursor (newest-first per page).
// Each page uses `before=oldestSigInPreviousPage` to walk backwards until
// we exhaust the window. This guarantees complete coverage regardless of volume.
var sigs []helius.SigEntry
before := ""
for {
batch, err := h.GetSignaturesForAddress(ctx, feeAccount, sigLimit, cursor.LastSig, before)
if err != nil {
return fmt.Errorf("get sigs (before=%s): %w", before, err)
}
sigs = append(sigs, batch...)
if len(batch) < sigLimit {
break // last page
}
before = batch[len(batch)-1].Signature
time.Sleep(300 * time.Millisecond) // respect Helius free-tier rate limit between pages
}
if len(sigs) == 0 {
return nil
}
// On incremental polls (cursor set), hitting the cap means we're dropping older
// txs from this window — tx counts will be understated, though fee stats remain
// a representative sample of the most-recent transactions.
// The initial bootstrap (no cursor) always hits the cap; that's expected.
if len(sigs) == sigLimit && cursor.LastSig != "" {
log.Printf("collector: %s: WARNING hit sig limit (%d) — older txs in this poll window dropped; reduce POLL_INTERVAL or increase limit", plt, sigLimit)
if len(sigs) > sigLimit {
log.Printf("collector: %s: paginated %d raw sigs (%d pages)", plt, len(sigs), (len(sigs)+sigLimit-1)/sigLimit)
}

// Reverse to process oldest-first so cursor is always the true watermark.
Expand Down
13 changes: 8 additions & 5 deletions harnesses/solana-exec/internal/helius/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,21 @@ type SigEntry struct {
Err any `json:"err"` // nil = success
}

// GetSignaturesForAddress fetches up to `limit` finalized signatures
// for `address`. Pass `until=""` to get the most recent; pass a sig
// to get only transactions newer than that sig (exclusive).
// Results are returned newest-first.
func (c *Client) GetSignaturesForAddress(ctx context.Context, address string, limit int, until string) ([]SigEntry, error) {
// GetSignaturesForAddress fetches up to `limit` finalized signatures for
// `address`, newest-first. `until` is the exclusive upper bound (cursor);
// `before` is the exclusive lower bound used for pagination (pass "" for
// the first page).
func (c *Client) GetSignaturesForAddress(ctx context.Context, address string, limit int, until, before string) ([]SigEntry, error) {
params := map[string]any{
"limit": limit,
"commitment": "finalized",
}
if until != "" {
params["until"] = until
}
if before != "" {
params["before"] = before
}

body, _ := json.Marshal(map[string]any{
"jsonrpc": "2.0",
Expand Down
Loading