diff --git a/.hermes/plans/2026-04-20_165747-hermes-integration.md b/.hermes/plans/2026-04-20_165747-hermes-integration.md deleted file mode 100644 index df9d11f5..00000000 --- a/.hermes/plans/2026-04-20_165747-hermes-integration.md +++ /dev/null @@ -1,336 +0,0 @@ -# Hermes integration proposal for augr - -## Goal - -Integrate Hermes with the existing `~/.agents` hub and the `augr` repo in a way that is low-risk, operationally useful, and consistent with the hub's documented boundary: - -- shared, human-managed assets live in `~/.agents` -- mutable runtime state stays native to each tool -- augr consumes the hub for workspace bootstrapping rather than inventing a parallel agent layout - -## What I found - -### In `augr` - -- The repo already expects the shared agent hub and documents a `~/.agents`-based workflow in `README.md`. -- `Taskfile.yml` defines: - - `task workspace` - - `task workspace:research` - - `task workspace:review` - - `task workspace:ops` -- But those tasks call `./scripts/workspace.sh`, and that file is missing. -- The actual hub workspace launcher exists in `~/.agents/scripts/bootstrap_tmux_workspace.sh`. -- The augr backend already exposes the right operator-facing surfaces for Hermes: - - `GET /api/v1/runs/{id}` - - `GET /api/v1/runs/{id}/decisions` - - `GET /api/v1/runs/{id}/snapshot` - - `GET /api/v1/events` - - `GET /api/v1/memories` - - `POST /api/v1/memories/search` - - `GET/POST /api/v1/conversations` - - `GET/POST /api/v1/conversations/{id}/messages` -- `internal/service/conversation.go` already builds LLM context from decisions, snapshots, and memories, which means augr already has a good "ask the agent why it did that" surface. - -### In `~/.agents` - -- The hub has a clear pattern for adding managed harnesses via `agents//agent.yaml`. -- The documented strategies are: - - `symlink-subpaths` - - `template-or-copy` - - `docs-only` - - `native-only` -- The hub script currently opens tmux windows for: - - `edit` - - `deck` - - `claude` - - `opencode` - - `db` - - `ops` -- Agent metadata is validated by schema and hub doctor tooling. -- The hub explicitly warns against centralizing mutable runtime state. - -## Recommendation - -Use a two-layer integration: - -1. `~/.agents` integration for developer workflow -2. augr API integration for operator workflow - -Do **not** put Hermes directly into augr's trading runtime first. - -That would be the wrong first move because it touches execution-critical code (`internal/agent`, runner wiring, risk flow, provider routing) when augr already exposes a safer control-plane interface for investigation and operations. - -## Proposed architecture - -### Layer 1: Add Hermes as a managed harness in `~/.agents` - -Add a new registry entry: - -- `~/.agents/agents/hermes/agent.yaml` - -Recommended initial posture: - -- `hub_strategy: native-only` or `docs-only` -- keep Hermes runtime state outside the hub -- use the hub only for: - - docs - - prompts - - launcher integration - - discoverability in hub doctor / bootstrap reporting - -Why this posture: - -- it matches the hub rule: centralize durable assets, not runtime state -- it avoids guessing at Hermes config semantics too early -- it gets Hermes into the standard tooling surface immediately - -Suggested initial metadata shape: - -```yaml -name: hermes -cli: hermes -config: ~/.hermes -runtime_roots: - - ~/.hermes - - ~/Documents/hermes -config_link: configs/hermes -binary: hermes -binary_type: cli -description: Hermes CLI agent and automation runtime -install_hint: Install Hermes CLI and ensure `hermes` is on PATH. -maturity: experimental -supports_mcp: false -optional: true -hub_strategy: native-only -config_required: true -shared_assets: - - docs - - prompts -bootstrap: - check_paths: - - ~/.hermes - - ~/Documents/hermes - notes: - - Keep Hermes runtime state native; do not centralize sessions, caches, or credentials in the hub. -validated_platforms: - - linux -``` - -Notes: - -- I would start with `supports_mcp: false` unless Hermes has a documented MCP contract you want the hub to validate. -- If Hermes later grows a stable rendered-config surface, you can move from `native-only` to `template-or-copy`. - -### Layer 2: Make augr launch Hermes from the shared workspace - -Fix the current broken workspace path in augr. - -Current problem: - -- `Taskfile.yml` points to `./scripts/workspace.sh` -- that file does not exist - -Proposed fix: - -- add `Code/projects/augr/scripts/workspace.sh` as a thin wrapper over the hub launcher -- wrapper should call: - -```bash -~/.agents/scripts/bootstrap_tmux_workspace.sh "$(pwd)" "${1:-$(basename "$(pwd)")}" -``` - -Then extend the shared hub launcher to support Hermes as an optional extra window: - -- add env vars like: - - `HERMES_BIN=${HERMES_BIN:-hermes}` - - `ENABLE_HERMES_WINDOW=${ENABLE_HERMES_WINDOW:-1}` -- if Hermes is installed, open a `hermes` tmux window after `opencode` - -Result: - -- augr keeps using the shared hub model -- Hermes becomes part of the standard repo workspace -- no augr-specific hardcoding of Hermes internals is needed - -## How Hermes should talk to augr - -Hermes should integrate with augr as an operator/control-plane client first, not as a runtime trading role. - -### Phase A: operator assistant over augr APIs - -Create a small Hermes-side augr integration surface that can: - -- authenticate with augr using API key or login flow -- list and inspect strategy runs -- fetch run decisions and snapshots -- search memories -- read events -- open or continue agent conversations -- trigger manual actions: - - run strategy - - cancel run - - inspect risk status - - toggle kill switch only with explicit operator confirmation - -This can be delivered as either: - -1. a Hermes skill/documented workflow, or -2. a lightweight augr client script/CLI wrapper consumed by Hermes - -I recommend starting with a thin client wrapper because augr already has stable HTTP surfaces. - -### Minimum useful endpoints for Hermes - -Read-only: - -- `GET /api/v1/strategies` -- `GET /api/v1/runs` -- `GET /api/v1/runs/{id}` -- `GET /api/v1/runs/{id}/decisions` -- `GET /api/v1/runs/{id}/snapshot` -- `GET /api/v1/events` -- `GET /api/v1/memories` -- `POST /api/v1/memories/search` -- `GET /api/v1/conversations` -- `GET /api/v1/conversations/{id}/messages` - -Write actions: - -- `POST /api/v1/conversations` -- `POST /api/v1/conversations/{id}/messages` -- `POST /api/v1/strategies/{id}/run` -- `POST /api/v1/runs/{id}/cancel` -- `POST /api/v1/risk/killswitch` - -### Why this is the right first integration - -Because augr already stores and exposes the exact artifacts Hermes needs to be useful: - -- decisions -- snapshots -- events -- memories -- conversations - -That means Hermes can act as: - -- operator copilot -- run investigator -- postmortem assistant -- risk review assistant - -without changing augr's execution pipeline. - -## Repo changes I would make first - -### In `~/.agents` - -1. Add `agents/hermes/agent.yaml` -2. Add `docs/HERMES.md` or similar usage doc -3. Update `scripts/bootstrap_tmux_workspace.sh` to optionally open a `hermes` window -4. Update validation/docs if needed - -### In `augr` - -1. Add `scripts/workspace.sh` wrapper so existing `task workspace*` commands actually work -2. Add `docs/hermes-integration.md` with: - - auth setup - - common Hermes workflows - - safe actions vs dangerous actions -3. Optionally add a small helper script, e.g.: - - `scripts/hermes-augr.sh` - - or `scripts/hermes_api.py` - -### Optional UI addition later - -In augr web UI, add a "Open in Hermes" affordance from: - -- run detail page -- decision timeline -- conversation page - -That action could: - -- copy a run-focused prompt -- open a local URL/command bridge if you later build one -- or simply render a ready-to-paste Hermes command block - -## Concrete first milestone - -If I were implementing this in the least risky order, I would do: - -1. Fix augr workspace launcher by adding `scripts/workspace.sh` -2. Register Hermes in `~/.agents/agents/hermes/agent.yaml` -3. Extend the hub tmux launcher with an optional `hermes` window -4. Add an augr helper script for read-only inspection: - - get run - - get decisions - - get snapshot - - search memories - - list conversations -5. Add a short repo doc showing example Hermes operator workflows - -That gives immediate value with minimal blast radius. - -## What I would not do first - -I would not start by: - -- adding Hermes as a new trading/runtime role inside `internal/agent` -- replacing augr's LLM provider layer with Hermes -- routing execution decisions through Hermes -- storing Hermes runtime state inside `~/.agents` - -Those are higher-risk and solve the wrong problem first. - -## Validation - -### Hub validation - -After adding Hermes to `~/.agents`: - -```bash -cd ~/.agents -python3 scripts/validate_agent_yaml.py -python3 scripts/validate_hub_metadata.py -python3 scripts/hub_doctor.py -bash scripts/validate-hub.sh -``` - -### augr validation - -After adding the launcher wrapper: - -```bash -cd ~/Code/projects/augr -task workspace -``` - -Expected result: - -- tmux session opens successfully -- standard windows appear -- Hermes window appears when enabled and installed - -### augr API integration validation - -Verify Hermes-side read-only flows against: - -- runs -- decisions -- snapshot -- memories search -- conversations - -before enabling write actions. - -## Final recommendation - -Best path: - -- integrate Hermes into `~/.agents` as a first-class harness -- fix augr to use the shared hub launcher it already expects -- use augr's existing API/conversation/memory surfaces so Hermes acts as an operator assistant first -- defer deep runtime integration until the workflow proves valuable - -This gives you a practical Hermes-on-augr workflow quickly, while staying aligned with both augr's current architecture and the hub's documented boundaries. diff --git a/.hermes/plans/2026-04-22-alpaca-reconciliation.md b/.hermes/plans/2026-04-22-alpaca-reconciliation.md new file mode 100644 index 00000000..ccf4fbb7 --- /dev/null +++ b/.hermes/plans/2026-04-22-alpaca-reconciliation.md @@ -0,0 +1,42 @@ +# Alpaca reconciliation implementation plan + +Goal: add a real Alpaca broker reconciliation path that imports Alpaca paper/live positions, orders, and fills into augr’s local positions/orders/trades tables and exposes it as an automation job. + +Architecture: introduce a dedicated reconciliation service under internal/automation that talks to Alpaca through a narrow snapshot interface, maps broker state into domain orders/positions/trades, and upserts local records with deterministic matching. Wire that service into the automation orchestrator as a new manual/scheduled job and add the minimal schema/repository support needed for idempotent fill tracking and reliable order matching. + +Tech stack: Go, existing Alpaca HTTP client, existing postgres repositories, automation orchestrator, SQL migrations, Go tests. + +Implementation slices: +1. Add failing unit tests for reconciliation service behavior and orchestrator registration. +2. Add schema support for durable Alpaca reconciliation metadata and bump required schema version. +3. Add repository capabilities needed for lookup by external order id and fill activity id. +4. Implement Alpaca snapshot fetch methods for orders and fills. +5. Implement reconciliation service. +6. Register/wire automation job in runtime. +7. Run targeted and broader verification. + +Planned files to touch: +- create: internal/automation/alpaca_reconciliation.go +- create: migrations/000032_alpaca_reconciliation.up.sql +- create: migrations/000032_alpaca_reconciliation.down.sql +- create: migrations/alpaca_reconciliation_migration_test.go +- modify: internal/automation/alpaca_reconciliation_test.go +- modify: internal/automation/orchestrator.go +- modify: internal/automation/jobs_premarket.go or another automation job file for registration +- modify: internal/execution/alpaca/broker.go +- modify: internal/execution/alpaca/broker_test.go +- modify: internal/repository/interfaces.go +- modify: internal/repository/postgres/order.go +- modify: internal/repository/postgres/order_test.go +- modify: internal/repository/postgres/trade.go +- modify: internal/repository/postgres/trade_test.go +- modify: internal/repository/postgres/schema_version.go +- modify: cmd/tradingagent/runtime.go +- modify: cmd/tradingagent/schema_version_sync_test.go + +Notes: +- Use strict TDD: write failing tests first, then minimal implementation. +- Keep behavior idempotent: repeated reconcile runs must not duplicate orders, positions, or fills. +- Prefer matching active strategies by ticker for imported positions/orders, but tolerate no matching strategy by leaving StrategyID nil. +- Fill dedupe should use Alpaca activity id, not just timestamp/qty/price. +- Existing DB currently cannot persist a broker-origin fill activity id, so migration will likely add one field to trades and maybe a unique index. diff --git a/.task/checksum/build b/.task/checksum/build new file mode 100644 index 00000000..63fc4f26 --- /dev/null +++ b/.task/checksum/build @@ -0,0 +1 @@ +35c142b74b47c572ca4c4dca6eb1a64a diff --git a/cmd/tradingagent/runtime.go b/cmd/tradingagent/runtime.go index 8f6509cb..a302cd7b 100644 --- a/cmd/tradingagent/runtime.go +++ b/cmd/tradingagent/runtime.go @@ -34,6 +34,7 @@ import ( "github.com/PatrickFanella/get-rich-quick/internal/discovery" "github.com/PatrickFanella/get-rich-quick/internal/domain" "github.com/PatrickFanella/get-rich-quick/internal/execution" + alpacaexecution "github.com/PatrickFanella/get-rich-quick/internal/execution/alpaca" "github.com/PatrickFanella/get-rich-quick/internal/execution/paper" "github.com/PatrickFanella/get-rich-quick/internal/llm" "github.com/PatrickFanella/get-rich-quick/internal/llm/anthropic" @@ -241,6 +242,25 @@ func newAPIServer(ctx context.Context, cfg config.Config, logger *slog.Logger) ( dataService := data.NewDataService(cfg, reg, marketDataCacheRepo, logger, socialTriage) deps.DataService = dataService + var alpacaReconciler *automation.AlpacaReconciler + if strings.TrimSpace(cfg.Brokers.Alpaca.APIKey) != "" && strings.TrimSpace(cfg.Brokers.Alpaca.APISecret) != "" { + alpacaClient := alpacaexecution.NewClient( + cfg.Brokers.Alpaca.APIKey, + cfg.Brokers.Alpaca.APISecret, + cfg.Brokers.Alpaca.PaperMode, + logger, + ) + alpacaReconciler = automation.NewAlpacaReconciler(automation.AlpacaReconcilerDeps{ + Broker: automation.NewAlpacaClientAdapter(alpacaClient), + StrategyRepo: strategyRepo, + OrderRepo: orderRepo, + PositionRepo: positionRepo, + TradeRepo: tradeRepo, + AuditLogRepo: auditLogRepo, + Logger: logger, + }) + deps.AlpacaReconciler = alpacaReconciler + } // Options data chain: Tradier (full Greeks from ORATS) → Yahoo (free, BS Greeks) // → Alpaca (paper account) → Polygon (rate-limited). optProviders := []data.OptionsDataProvider{} @@ -338,6 +358,7 @@ func newAPIServer(ctx context.Context, cfg config.Config, logger *slog.Logger) ( Universe: deps.Universe, Polygon: polygonClientForAuto, DataService: dataService, + AlpacaReconciler: alpacaReconciler, OptionsProvider: deps.OptionsProvider, LLMProvider: deps.LLMProvider, EmbeddingProvider: embeddingProvider, diff --git a/cmd/tradingagent/runtime_test.go b/cmd/tradingagent/runtime_test.go index e09ec406..23e86ec2 100644 --- a/cmd/tradingagent/runtime_test.go +++ b/cmd/tradingagent/runtime_test.go @@ -14,6 +14,7 @@ import ( "github.com/PatrickFanella/get-rich-quick/internal/agent" "github.com/PatrickFanella/get-rich-quick/internal/api" + "github.com/PatrickFanella/get-rich-quick/internal/automation" "github.com/PatrickFanella/get-rich-quick/internal/config" "github.com/PatrickFanella/get-rich-quick/internal/data" "github.com/PatrickFanella/get-rich-quick/internal/domain" @@ -69,7 +70,7 @@ func TestNewAPIServerSchemaBehindFailsFast(t *testing.T) { if mismatchErr.Required != pgrepo.RequiredSchemaVersion { t.Fatalf("mismatchErr.Required = %d, want %d", mismatchErr.Required, pgrepo.RequiredSchemaVersion) } - for _, want := range []string{"current version 29", "required version 30", "run migrations, then restart the process", "fresh process restart"} { + for _, want := range []string{"current version 31", "required version 32", "run migrations, then restart the process", "fresh process restart"} { if !strings.Contains(err.Error(), want) { t.Fatalf("error %q missing %q", err.Error(), want) } @@ -123,7 +124,7 @@ func TestNewAPIServerSchemaAheadFailsFast(t *testing.T) { if mismatchErr.Required != pgrepo.RequiredSchemaVersion { t.Fatalf("mismatchErr.Required = %d, want %d", mismatchErr.Required, pgrepo.RequiredSchemaVersion) } - for _, want := range []string{"current version 31", "required version 30", "run migrations, then restart the process", "fresh process restart"} { + for _, want := range []string{"current version 33", "required version 32", "run migrations, then restart the process", "fresh process restart"} { if !strings.Contains(err.Error(), want) { t.Fatalf("error %q missing %q", err.Error(), want) } @@ -244,6 +245,79 @@ func TestNewAPIServerSchemaDBUnreachableFailsBeforeSchemaGate(t *testing.T) { } } +func TestNewAPIServerWiresAlpacaReconcileAutomationJob(t *testing.T) { + origNewDB := runtimeNewDB + origCurrentSchemaVersion := runtimeCurrentSchemaVersion + origAfterSchemaGate := runtimeAfterSchemaGate + origCloseDB := runtimeCloseDB + origNewServer := runtimeNewServer + defer func() { + runtimeNewDB = origNewDB + runtimeCurrentSchemaVersion = origCurrentSchemaVersion + runtimeAfterSchemaGate = origAfterSchemaGate + runtimeCloseDB = origCloseDB + runtimeNewServer = origNewServer + }() + + pool, err := pgxpool.New(context.Background(), "postgres://postgres:***@127.0.0.1:1/postgres?sslmode=disable&connect_timeout=1") + if err != nil { + t.Fatalf("pgxpool.New() error = %v", err) + } + defer pool.Close() + + var capturedDeps api.Deps + var cleanupCalled atomic.Bool + runtimeNewDB = func(context.Context, string) (*pgrepo.DB, error) { + return &pgrepo.DB{Pool: pool}, nil + } + runtimeCurrentSchemaVersion = func(context.Context, *pgxpool.Pool) (int, error) { + return pgrepo.RequiredSchemaVersion, nil + } + runtimeAfterSchemaGate = func() {} + runtimeCloseDB = func(*pgrepo.DB) { cleanupCalled.Store(true) } + runtimeNewServer = func(_ api.ServerConfig, deps api.Deps, _ *slog.Logger) (*api.Server, error) { + capturedDeps = deps + return &api.Server{}, nil + } + + cfg := config.Config{ + Environment: "development", + Database: config.DatabaseConfig{URL: "postgres://ignored"}, + Features: config.FeatureFlags{ + EnableScheduler: true, + EnableTickerDiscovery: true, + }, + DataProviders: config.DataProviderConfigs{ + Polygon: config.DataProviderConfig{APIKey: "polygon-key"}, + }, + Brokers: config.BrokerConfigs{ + Alpaca: config.BrokerConfig{APIKey: "alpaca-key", APISecret: "alpaca-secret", PaperMode: true}, + }, + Embedding: config.EmbeddingConfig{Model: "nomic-embed-text", Timeout: time.Second}, + LLM: config.LLMConfig{Providers: config.LLMProviderConfigs{Ollama: config.OllamaConfig{BaseURL: "http://localhost:11434"}}}, + } + + _, _, cleanup, err := newAPIServer(context.Background(), cfg, slogDiscardLogger()) + if err != nil { + t.Fatalf("newAPIServer() error = %v", err) + } + if capturedDeps.Automation == nil { + t.Fatal("newAPIServer() automation = nil, want non-nil") + } + status := runtimeSingleAutomationJobStatus(t, capturedDeps.Automation, "alpaca_reconcile") + if status.Name != "alpaca_reconcile" { + t.Fatalf("status.Name = %q, want alpaca_reconcile", status.Name) + } + if got := status.Schedule; got == "" || got == "Manual only" { + t.Fatalf("status.Schedule = %q, want scheduled job description", got) + } + + cleanup() + if !cleanupCalled.Load() { + t.Fatal("cleanup did not close db") + } +} + func TestNewNotificationManager_DiscordAlertDispatch(t *testing.T) { t.Parallel() @@ -358,6 +432,17 @@ type stubDecisionRepo struct { decisions []domain.AgentDecision } +func runtimeSingleAutomationJobStatus(t *testing.T, orch *automation.JobOrchestrator, jobName string) automation.JobStatus { + t.Helper() + for _, status := range orch.Status() { + if status.Name == jobName { + return status + } + } + t.Fatalf("job status %q not found", jobName) + return automation.JobStatus{} +} + type captureProvider struct{} func (captureProvider) Complete(_ context.Context, request llm.CompletionRequest) (*llm.CompletionResponse, error) { diff --git a/internal/agent/runner.go b/internal/agent/runner.go index ae797497..305abd66 100644 --- a/internal/agent/runner.go +++ b/internal/agent/runner.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log/slog" + "runtime/debug" "sync" "time" @@ -233,12 +234,75 @@ func (r *Runner) RunStrategy(ctx context.Context, strategy domain.Strategy, glob } // Run executes one prepared run and returns the canonical result. -func (r *Runner) Run(ctx context.Context, prepared PreparedRun) (*RunResult, error) { - slog.Info("DEBUG: runner.Run entered", slog.String("run_id", prepared.RunID.String())) +func (r *Runner) Run(ctx context.Context, prepared PreparedRun) (result *RunResult, runErr error) { if r.persister == nil { return nil, fmt.Errorf("agent/runner: persister is required") } + var ( + run domain.PipelineRun + state *PipelineState + phaseTimings map[string]int64 + warnings []RunWarning + cacheStatsCollector *llm.CacheStatsCollector + ) + + defer func() { + recovered := recover() + if recovered == nil { + return + } + + panicErr := fmt.Errorf("agent/runner: panic recovered: %v", recovered) + r.logger.Error("agent/runner: recovered panic", + slog.Any("panic", recovered), + slog.String("stack", string(debug.Stack())), + ) + + completedAt := r.currentTime().UTC() + phaseTimingsJSON, _ := json.Marshal(phaseTimings) + if run.ID != uuid.Nil { + _ = r.persister.RecordRunComplete( + context.Background(), + run.ID, + run.TradeDate, + domain.PipelineStatusFailed, + completedAt, + panicErr.Error(), + phaseTimingsJSON, + ) + } + if cacheStatsCollector != nil { + r.helper.emitCacheStats(state, cacheStatsCollector, run.ID, prepared.Strategy.ID, prepared.Strategy.Ticker) + } + if run.ID != uuid.Nil { + r.helper.persistStructuredTerminalEvent(r.helper.newStructuredEvent( + run.ID, + prepared.Strategy.ID, + AgentEventKindPipelineFailed, + "", + "Pipeline failed", + panicErr.Error(), + map[string]any{"phase": "panic", "error_message": panicErr.Error()}, + []string{"pipeline", "failed"}, + )) + } + r.helper.emitEvent(PipelineEvent{ + Type: PipelineError, + PipelineRunID: run.ID, + StrategyID: prepared.Strategy.ID, + Ticker: prepared.Strategy.Ticker, + Error: panicErr.Error(), + OccurredAt: r.currentTime().UTC(), + }) + + run.Status = domain.PipelineStatusFailed + run.CompletedAt = &completedAt + run.ErrorMessage = panicErr.Error() + result = &RunResult{Run: run, Signal: r.canonicalSignal(state), State: snapshotState(state), Warnings: warnings} + runErr = panicErr + }() + var cancel context.CancelFunc if prepared.Runtime.PipelineTimeout > 0 { ctx, cancel = context.WithTimeout(ctx, prepared.Runtime.PipelineTimeout) @@ -247,11 +311,11 @@ func (r *Runner) Run(ctx context.Context, prepared PreparedRun) (*RunResult, err } defer cancel() - cacheStatsCollector := llm.NewCacheStatsCollector() + cacheStatsCollector = llm.NewCacheStatsCollector() ctx = llm.WithCacheStatsCollector(ctx, cacheStatsCollector) now := r.currentTime().UTC() - run := domain.PipelineRun{ + run = domain.PipelineRun{ ID: uuid.New(), StrategyID: prepared.Strategy.ID, Ticker: prepared.Strategy.Ticker, @@ -270,17 +334,15 @@ func (r *Runner) Run(ctx context.Context, prepared PreparedRun) (*RunResult, err if r.runRegistry != nil { r.runRegistry.Register(run.ID, cancel) defer r.runRegistry.Deregister(run.ID) - slog.Info("DEBUG: runRegistry registered", slog.String("run_id", run.ID.String())) } - state := &PipelineState{ + state = &PipelineState{ PipelineRunID: run.ID, StrategyID: prepared.Strategy.ID, Ticker: prepared.Strategy.Ticker, mu: &sync.Mutex{}, } applyInitialStateSeed(state, prepared.InitialState) - slog.Info("DEBUG: initial state seeded, about to persist PipelineStarted", slog.String("run_id", run.ID.String())) r.helper.persistStructuredEvent(ctx, r.helper.newStructuredEvent( run.ID, @@ -300,8 +362,8 @@ func (r *Runner) Run(ctx context.Context, prepared PreparedRun) (*RunResult, err OccurredAt: r.currentTime().UTC(), }) - phaseTimings := map[string]int64{} - warnings := make([]RunWarning, 0) + phaseTimings = map[string]int64{} + warnings = make([]RunWarning, 0) var warningsMu sync.Mutex phases := []struct { name string @@ -356,7 +418,9 @@ func (r *Runner) Run(ctx context.Context, prepared PreparedRun) (*RunResult, err run.Status = domain.PipelineStatusFailed run.CompletedAt = &completedAt run.ErrorMessage = err.Error() - return &RunResult{Run: run, Signal: r.canonicalSignal(state), State: snapshotState(state), Warnings: warnings}, err + result = &RunResult{Run: run, Signal: r.canonicalSignal(state), State: snapshotState(state), Warnings: warnings} + runErr = err + return } phaseTimings[phase.name+"_ms"] = time.Since(phaseStart).Milliseconds() r.helper.persistStructuredEvent(ctx, r.helper.newStructuredEvent( @@ -395,7 +459,9 @@ func (r *Runner) Run(ctx context.Context, prepared PreparedRun) (*RunResult, err run.Status = domain.PipelineStatusCompleted run.CompletedAt = &completedAt - return &RunResult{Run: run, Signal: r.canonicalSignal(state), State: snapshotState(state), Warnings: warnings}, nil + result = &RunResult{Run: run, Signal: r.canonicalSignal(state), State: snapshotState(state), Warnings: warnings} + runErr = nil + return } func applyInitialStateSeed(state *PipelineState, seed InitialStateSeed) { diff --git a/internal/agent/runner_test.go b/internal/agent/runner_test.go index 89f78218..10393082 100644 --- a/internal/agent/runner_test.go +++ b/internal/agent/runner_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "sort" + "strings" "sync" "testing" "time" @@ -565,3 +566,49 @@ func TestRunnerRun_ContextCancellation(t *testing.T) { t.Errorf("run status = %s, want failed", result.Run.Status) } } + +func TestRunnerRun_PanicInPhaseMarksRunFailed(t *testing.T) { + t.Parallel() + + def := defaultRunnerDefinition() + def.Trader = stubTradeAgent{name: "trader", role: AgentRoleTrader, fn: func(context.Context, TradingInput) (TradingOutput, error) { + panic("boom panic") + }} + + persister := newRunnerSpyPersister() + events := make(chan PipelineEvent, 64) + runner := NewRunner(def, Dependencies{Persister: persister, Events: events}) + + prepared, err := runner.Prepare(strategyWithDebateRounds(t, "TEST", 1), GlobalSettings{}) + if err != nil { + t.Fatalf("Prepare() error = %v", err) + } + + result, runErr := runner.Run(context.Background(), prepared) + if runErr == nil { + t.Fatal("Run() error = nil, want panic recovery error") + } + if !strings.Contains(runErr.Error(), "panic recovered") { + t.Fatalf("Run() error = %q, want panic recovered substring", runErr.Error()) + } + if result == nil { + t.Fatal("Run() result = nil, want failed result") + } + if result.Run.Status != domain.PipelineStatusFailed { + t.Fatalf("run status = %s, want failed", result.Run.Status) + } + if !strings.Contains(result.Run.ErrorMessage, "panic recovered") { + t.Fatalf("run error_message = %q, want panic recovered substring", result.Run.ErrorMessage) + } + + close(events) + pipelineErrors := 0 + for event := range events { + if event.Type == PipelineError { + pipelineErrors++ + } + } + if pipelineErrors == 0 { + t.Fatal("expected at least one PipelineError event after panic recovery") + } +} diff --git a/internal/api/automation_alpaca_handlers.go b/internal/api/automation_alpaca_handlers.go new file mode 100644 index 00000000..680876bc --- /dev/null +++ b/internal/api/automation_alpaca_handlers.go @@ -0,0 +1,49 @@ +package api + +import ( + "context" + "net/http" + + "github.com/PatrickFanella/get-rich-quick/internal/automation" +) + +type AlpacaAutomationReconciler interface { + Reconcile(ctx context.Context) (automation.AlpacaReconcileSummary, error) + Verify(ctx context.Context) (automation.AlpacaVerificationReport, error) +} + +type AlpacaReconcileResponse struct { + Summary automation.AlpacaReconcileSummary `json:"summary"` + Verification automation.AlpacaVerificationReport `json:"verification"` +} + +func (s *Server) handleRunAlpacaReconcile(w http.ResponseWriter, r *http.Request) { + if s.alpacaReconciler == nil { + respondError(w, http.StatusServiceUnavailable, "alpaca reconciliation not configured", ErrCodeInternal) + return + } + summary, err := s.alpacaReconciler.Reconcile(r.Context()) + if err != nil { + respondError(w, http.StatusBadGateway, err.Error(), ErrCodeInternal) + return + } + verification, err := s.alpacaReconciler.Verify(r.Context()) + if err != nil { + respondError(w, http.StatusBadGateway, err.Error(), ErrCodeInternal) + return + } + respondJSON(w, http.StatusOK, AlpacaReconcileResponse{Summary: summary, Verification: verification}) +} + +func (s *Server) handleVerifyAlpacaReconcile(w http.ResponseWriter, r *http.Request) { + if s.alpacaReconciler == nil { + respondError(w, http.StatusServiceUnavailable, "alpaca reconciliation not configured", ErrCodeInternal) + return + } + report, err := s.alpacaReconciler.Verify(r.Context()) + if err != nil { + respondError(w, http.StatusBadGateway, err.Error(), ErrCodeInternal) + return + } + respondJSON(w, http.StatusOK, report) +} diff --git a/internal/api/automation_alpaca_handlers_test.go b/internal/api/automation_alpaca_handlers_test.go new file mode 100644 index 00000000..bb34713d --- /dev/null +++ b/internal/api/automation_alpaca_handlers_test.go @@ -0,0 +1,81 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/PatrickFanella/get-rich-quick/internal/automation" +) + +type stubAlpacaAdminReconciler struct { + summary automation.AlpacaReconcileSummary + report automation.AlpacaVerificationReport + err error + calls int +} + +func (s *stubAlpacaAdminReconciler) Reconcile(ctx context.Context) (automation.AlpacaReconcileSummary, error) { + s.calls++ + return s.summary, s.err +} + +func (s *stubAlpacaAdminReconciler) Verify(ctx context.Context) (automation.AlpacaVerificationReport, error) { + return s.report, s.err +} + +func TestRunAlpacaReconcileNowReturnsSummaryAndVerification(t *testing.T) { + t.Parallel() + + reconciler := &stubAlpacaAdminReconciler{ + summary: automation.AlpacaReconcileSummary{ + OrdersCreated: 2, + OrdersUpdated: 1, + PositionsCreated: 1, + TradesCreated: 3, + }, + report: automation.AlpacaVerificationReport{ + OrdersChecked: 3, + PositionsChecked: 1, + FillsChecked: 3, + }, + } + s := &Server{alpacaReconciler: reconciler} + + req := httptest.NewRequest(http.MethodPost, "/api/v1/automation/alpaca/reconcile", nil) + rr := httptest.NewRecorder() + s.handleRunAlpacaReconcile(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rr.Code) + } + + var resp AlpacaReconcileResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.Summary.OrdersCreated != 2 { + t.Fatalf("OrdersCreated = %d, want 2", resp.Summary.OrdersCreated) + } + if resp.Verification.OrdersChecked != 3 { + t.Fatalf("OrdersChecked = %d, want 3", resp.Verification.OrdersChecked) + } + if reconciler.calls != 1 { + t.Fatalf("Reconcile calls = %d, want 1", reconciler.calls) + } +} + +func TestRunAlpacaReconcileNowRequiresReconciler(t *testing.T) { + t.Parallel() + + s := &Server{} + req := httptest.NewRequest(http.MethodPost, "/api/v1/automation/alpaca/reconcile", nil) + rr := httptest.NewRecorder() + s.handleRunAlpacaReconcile(rr, req) + + if rr.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503", rr.Code) + } +} diff --git a/internal/api/automation_handlers_test.go b/internal/api/automation_handlers_test.go index 4fb57b63..41924763 100644 --- a/internal/api/automation_handlers_test.go +++ b/internal/api/automation_handlers_test.go @@ -148,3 +148,44 @@ func TestAutomationHealthFailingJobsCount(t *testing.T) { t.Errorf("expected healthy=true (no job has >=3 consecutive failures)") } } + +func TestAutomationStatusIncludesAlpacaReconcileLastSummary(t *testing.T) { + t.Parallel() + + o := newTestOrchestrator() + registerJob(o, "alpaca_reconcile") + o.SetLastSummary("alpaca_reconcile", map[string]int{ + "orders_created": 2, + "positions_created": 1, + "trades_created": 3, + }) + + s := &Server{automation: o} + req := httptest.NewRequest(http.MethodGet, "/api/v1/automation/status", nil) + rr := httptest.NewRecorder() + s.handleGetAutomationStatus(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", rr.Code) + } + + var statuses []automation.JobStatus + if err := json.NewDecoder(rr.Body).Decode(&statuses); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(statuses) != 1 { + t.Fatalf("len(statuses) = %d, want 1", len(statuses)) + } + if statuses[0].Name != "alpaca_reconcile" { + t.Fatalf("status name = %q, want alpaca_reconcile", statuses[0].Name) + } + if statuses[0].LastSummary == nil { + t.Fatal("LastSummary = nil, want non-nil") + } + if statuses[0].LastSummary["orders_created"] != 2 { + t.Fatalf("orders_created = %d, want 2", statuses[0].LastSummary["orders_created"]) + } + if statuses[0].LastSummary["trades_created"] != 3 { + t.Fatalf("trades_created = %d, want 3", statuses[0].LastSummary["trades_created"]) + } +} diff --git a/internal/api/middleware.go b/internal/api/middleware.go index 44b5ffd5..a9c5a80d 100644 --- a/internal/api/middleware.go +++ b/internal/api/middleware.go @@ -238,6 +238,10 @@ func RequestLogger(logger *slog.Logger) func(http.Handler) http.Handler { next.ServeHTTP(sw, r) + if shouldSuppressRequestLog(r.URL.Path) { + return + } + logger.Info("http request", slog.String("method", r.Method), slog.String("path", r.URL.Path), @@ -248,6 +252,15 @@ func RequestLogger(logger *slog.Logger) func(http.Handler) http.Handler { } } +func shouldSuppressRequestLog(path string) bool { + switch path { + case "/health", "/healthz", "/metrics", "/api/v1/automation/status", "/api/v1/strategies": + return true + default: + return false + } +} + // statusCapture wraps http.ResponseWriter to record the status code. type statusCapture struct { http.ResponseWriter diff --git a/internal/api/request_logger_test.go b/internal/api/request_logger_test.go new file mode 100644 index 00000000..cad02ccb --- /dev/null +++ b/internal/api/request_logger_test.go @@ -0,0 +1,59 @@ +package api + +import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestRequestLoggerSkipsProbePaths(t *testing.T) { + t.Parallel() + + var logOutput bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&logOutput, nil)) + handler := RequestLogger(logger)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + for _, path := range []string{"/health", "/healthz", "/metrics", "/api/v1/automation/status", "/api/v1/strategies"} { + req := httptest.NewRequest(http.MethodGet, path, nil) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("request to %s returned status %d, want 200", path, rr.Code) + } + } + + if strings.TrimSpace(logOutput.String()) != "" { + t.Fatalf("expected probe paths to be suppressed from logs, got: %s", logOutput.String()) + } +} + +func TestRequestLoggerLogsNonSuppressedPath(t *testing.T) { + t.Parallel() + + var logOutput bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&logOutput, nil)) + handler := RequestLogger(logger)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusAccepted) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/strategies/123", nil) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + if rr.Code != http.StatusAccepted { + t.Fatalf("request status = %d, want %d", rr.Code, http.StatusAccepted) + } + + out := logOutput.String() + if !strings.Contains(out, "http request") { + t.Fatalf("expected request log entry, got: %s", out) + } + if !strings.Contains(out, `"path":"/api/v1/strategies/123"`) { + t.Fatalf("expected path field in request log, got: %s", out) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 4773b0ca..b5fd06f2 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -62,7 +62,8 @@ type Server struct { universeRepo universe.UniverseRepository // Automation - automation *automation.JobOrchestrator + automation *automation.JobOrchestrator + alpacaReconciler AlpacaAutomationReconciler // Risk engine risk risk.RiskEngine @@ -184,6 +185,7 @@ type Deps struct { Universe *universe.Universe UniverseRepo universe.UniverseRepository Automation *automation.JobOrchestrator + AlpacaReconciler AlpacaAutomationReconciler NewsFeedRepo *pgrepo.NewsFeedRepo Risk risk.RiskEngine Settings SettingsService @@ -292,6 +294,7 @@ func NewServer(cfg ServerConfig, deps Deps, logger *slog.Logger) (*Server, error universe: deps.Universe, universeRepo: deps.UniverseRepo, automation: deps.Automation, + alpacaReconciler: deps.AlpacaReconciler, newsFeedRepo: deps.NewsFeedRepo, risk: deps.Risk, settings: settingsService, @@ -480,6 +483,8 @@ func NewServer(cfg ServerConfig, deps Deps, logger *slog.Logger) (*Server, error v1.Route("/automation", func(ar chi.Router) { ar.Get("/status", s.handleGetAutomationStatus) ar.Get("/health", s.handleGetAutomationHealth) + ar.Get("/alpaca/verify", s.handleVerifyAlpacaReconcile) + ar.Post("/alpaca/reconcile", s.handleRunAlpacaReconcile) ar.Post("/jobs/{name}/run", s.handleRunAutomationJob) ar.Post("/jobs/{name}/enable", s.handleSetAutomationJobEnabled) }) diff --git a/internal/automation/alpaca_client_adapter.go b/internal/automation/alpaca_client_adapter.go new file mode 100644 index 00000000..a334a5e5 --- /dev/null +++ b/internal/automation/alpaca_client_adapter.go @@ -0,0 +1,373 @@ +package automation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/url" + "strconv" + "strings" + "time" + + alpacaexec "github.com/PatrickFanella/get-rich-quick/internal/execution/alpaca" + "github.com/PatrickFanella/get-rich-quick/internal/domain" +) + +const alpacaActivitiesPageSize = 100 + +// AlpacaClientAdapter adapts the Alpaca execution client into reconciliation snapshots. +type AlpacaClientAdapter struct { + client *alpacaexec.Client + broker *alpacaexec.Broker + logger *slog.Logger +} + +type alpacaOrderResponse struct { + ID string `json:"id"` + Symbol string `json:"symbol"` + Side string `json:"side"` + Type string `json:"type"` + Qty string `json:"qty"` + LimitPrice string `json:"limit_price"` + StopPrice string `json:"stop_price"` + FilledQty string `json:"filled_qty"` + FilledAvgPrice string `json:"filled_avg_price"` + Status string `json:"status"` + SubmittedAt string `json:"submitted_at"` + FilledAt string `json:"filled_at"` +} + +type alpacaFillActivityResponse struct { + ActivityType string `json:"activity_type"` + ID string `json:"id"` + OrderID string `json:"order_id"` + Symbol string `json:"symbol"` + Side string `json:"side"` + Qty string `json:"qty"` + Price string `json:"price"` + TransactionTime string `json:"transaction_time"` + OrderStatus string `json:"order_status"` + Type string `json:"type"` +} + +func NewAlpacaClientAdapter(client *alpacaexec.Client) *AlpacaClientAdapter { + return &AlpacaClientAdapter{ + client: client, + broker: alpacaexec.NewBroker(client), + logger: slog.Default(), + } +} + +func (a *AlpacaClientAdapter) GetPositions(ctx context.Context) ([]domain.Position, error) { + if a == nil || a.broker == nil { + return nil, errors.New("alpaca: reconciliation broker is required") + } + return a.broker.GetPositions(ctx) +} + +func (a *AlpacaClientAdapter) ListOrders(ctx context.Context) ([]BrokerOrderSnapshot, error) { + if a == nil || a.client == nil { + return nil, errors.New("alpaca: reconciliation client is required") + } + + responseBody, err := a.client.Get(ctx, "/v2/orders", url.Values{ + "status": {"all"}, + "limit": {"500"}, + "direction": {"desc"}, + }) + if err != nil { + return nil, fmt.Errorf("alpaca: list orders: %w", err) + } + + var response []alpacaOrderResponse + if err := json.Unmarshal(responseBody, &response); err != nil { + return nil, fmt.Errorf("alpaca: decode orders response: %w", err) + } + + orders := make([]BrokerOrderSnapshot, 0, len(response)) + for _, raw := range response { + order, err := mapAlpacaOrderSnapshot(raw) + if err != nil { + return nil, err + } + orders = append(orders, order) + } + return orders, nil +} + +func (a *AlpacaClientAdapter) ListFills(ctx context.Context) ([]BrokerFillSnapshot, error) { + if a == nil || a.client == nil { + return nil, errors.New("alpaca: reconciliation client is required") + } + + var ( + fills []BrokerFillSnapshot + pageToken string + ) + for { + params := url.Values{ + "direction": {"desc"}, + "page_size": {strconv.Itoa(alpacaActivitiesPageSize)}, + } + if strings.TrimSpace(pageToken) != "" { + params.Set("page_token", pageToken) + } + + responseBody, err := a.client.Get(ctx, "/v2/account/activities/FILL", params) + if err != nil { + return nil, fmt.Errorf("alpaca: list fills: %w", err) + } + + var response []alpacaFillActivityResponse + if err := json.Unmarshal(responseBody, &response); err != nil { + return nil, fmt.Errorf("alpaca: decode fills response: %w", err) + } + if len(response) == 0 { + break + } + + for _, raw := range response { + fill, err := mapAlpacaFillSnapshot(raw) + if err != nil { + return nil, err + } + fills = append(fills, fill) + } + + if len(response) < alpacaActivitiesPageSize { + break + } + pageToken = response[len(response)-1].ID + } + return fills, nil +} + +func mapAlpacaOrderSnapshot(raw alpacaOrderResponse) (BrokerOrderSnapshot, error) { + externalID := strings.TrimSpace(raw.ID) + if externalID == "" { + return BrokerOrderSnapshot{}, errors.New("alpaca: order id is required") + } + ticker := strings.TrimSpace(raw.Symbol) + if ticker == "" { + return BrokerOrderSnapshot{}, errors.New("alpaca: order symbol is required") + } + side, err := mapAlpacaOrderSide(raw.Side) + if err != nil { + return BrokerOrderSnapshot{}, err + } + orderType, err := mapAlpacaOrderType(raw.Type) + if err != nil { + return BrokerOrderSnapshot{}, err + } + quantity, err := parseRequiredFloat("qty", raw.Qty) + if err != nil { + return BrokerOrderSnapshot{}, err + } + filledQty, err := parseRequiredFloat("filled_qty", raw.FilledQty) + if err != nil { + return BrokerOrderSnapshot{}, err + } + limitPrice, err := parseOptionalFloat("limit_price", raw.LimitPrice) + if err != nil { + return BrokerOrderSnapshot{}, err + } + stopPrice, err := parseOptionalFloat("stop_price", raw.StopPrice) + if err != nil { + return BrokerOrderSnapshot{}, err + } + filledAvgPrice, err := parseOptionalFloat("filled_avg_price", raw.FilledAvgPrice) + if err != nil { + return BrokerOrderSnapshot{}, err + } + status, err := mapAlpacaOrderStatus(raw.Status) + if err != nil { + return BrokerOrderSnapshot{}, err + } + submittedAt, err := parseOptionalTime("submitted_at", raw.SubmittedAt) + if err != nil { + return BrokerOrderSnapshot{}, err + } + filledAt, err := parseOptionalTime("filled_at", raw.FilledAt) + if err != nil { + return BrokerOrderSnapshot{}, err + } + + return BrokerOrderSnapshot{ + ExternalID: externalID, + Ticker: ticker, + Side: side, + OrderType: orderType, + Quantity: quantity, + LimitPrice: limitPrice, + StopPrice: stopPrice, + FilledQuantity: filledQty, + FilledAvgPrice: filledAvgPrice, + Status: status, + SubmittedAt: submittedAt, + FilledAt: filledAt, + Broker: "alpaca", + }, nil +} + +func mapAlpacaFillSnapshot(raw alpacaFillActivityResponse) (BrokerFillSnapshot, error) { + activityID := strings.TrimSpace(raw.ID) + if activityID == "" { + return BrokerFillSnapshot{}, errors.New("alpaca: fill activity id is required") + } + externalID := strings.TrimSpace(raw.OrderID) + if externalID == "" { + return BrokerFillSnapshot{}, errors.New("alpaca: fill order id is required") + } + ticker := strings.TrimSpace(raw.Symbol) + if ticker == "" { + return BrokerFillSnapshot{}, errors.New("alpaca: fill symbol is required") + } + side, err := mapAlpacaOrderSide(raw.Side) + if err != nil { + return BrokerFillSnapshot{}, err + } + quantity, err := parseRequiredFloat("qty", raw.Qty) + if err != nil { + return BrokerFillSnapshot{}, err + } + price, err := parseRequiredFloat("price", raw.Price) + if err != nil { + return BrokerFillSnapshot{}, err + } + executedAt, err := parseRequiredTime("transaction_time", raw.TransactionTime) + if err != nil { + return BrokerFillSnapshot{}, err + } + status, err := mapAlpacaFillOrderStatus(raw.OrderStatus, raw.Type) + if err != nil { + return BrokerFillSnapshot{}, err + } + + return BrokerFillSnapshot{ + ActivityID: activityID, + ExternalID: externalID, + Ticker: ticker, + Side: side, + Quantity: quantity, + Price: price, + ExecutedAt: executedAt, + OrderStatus: status, + Fee: 0, + }, nil +} + +func mapAlpacaOrderSide(raw string) (domain.OrderSide, error) { + switch side := strings.ToLower(strings.TrimSpace(raw)); side { + case string(domain.OrderSideBuy): + return domain.OrderSideBuy, nil + case string(domain.OrderSideSell): + return domain.OrderSideSell, nil + default: + return "", fmt.Errorf("alpaca: unsupported order side %q", raw) + } +} + +func mapAlpacaOrderType(raw string) (domain.OrderType, error) { + switch orderType := strings.ToLower(strings.TrimSpace(raw)); orderType { + case string(domain.OrderTypeMarket): + return domain.OrderTypeMarket, nil + case string(domain.OrderTypeLimit): + return domain.OrderTypeLimit, nil + case string(domain.OrderTypeStop): + return domain.OrderTypeStop, nil + case string(domain.OrderTypeStopLimit): + return domain.OrderTypeStopLimit, nil + case string(domain.OrderTypeTrailingStop): + return domain.OrderTypeTrailingStop, nil + default: + return "", fmt.Errorf("alpaca: unsupported order type %q", raw) + } +} + +func mapAlpacaOrderStatus(raw string) (domain.OrderStatus, error) { + switch status := strings.ToLower(strings.TrimSpace(raw)); status { + case "": + return "", errors.New("alpaca: order status is required") + case "accepted_for_bidding", "calculated", "held", "pending_cancel", "pending_new", "pending_replace": + return domain.OrderStatusPending, nil + case "accepted", "done_for_day", "new", "replaced", "stopped", "suspended": + return domain.OrderStatusSubmitted, nil + case "partially_filled": + return domain.OrderStatusPartial, nil + case "filled": + return domain.OrderStatusFilled, nil + case "canceled", "expired": + return domain.OrderStatusCancelled, nil + case "rejected": + return domain.OrderStatusRejected, nil + default: + return "", fmt.Errorf("alpaca: unsupported order status %q", raw) + } +} + +func mapAlpacaFillOrderStatus(rawStatus, rawType string) (domain.OrderStatus, error) { + if strings.TrimSpace(rawStatus) != "" { + return mapAlpacaOrderStatus(rawStatus) + } + switch fillType := strings.ToLower(strings.TrimSpace(rawType)); fillType { + case "partial_fill": + return domain.OrderStatusPartial, nil + case "fill": + return domain.OrderStatusFilled, nil + case "": + return "", errors.New("alpaca: fill order status is required") + default: + return "", fmt.Errorf("alpaca: unsupported fill type %q", rawType) + } +} + +func parseRequiredFloat(fieldName, value string) (float64, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return 0, fmt.Errorf("alpaca: %s is required", fieldName) + } + parsed, err := strconv.ParseFloat(trimmed, 64) + if err != nil { + return 0, fmt.Errorf("alpaca: parse %s: %w", fieldName, err) + } + return parsed, nil +} + +func parseOptionalFloat(fieldName, value string) (*float64, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil, nil + } + parsed, err := strconv.ParseFloat(trimmed, 64) + if err != nil { + return nil, fmt.Errorf("alpaca: parse %s: %w", fieldName, err) + } + return &parsed, nil +} + +func parseRequiredTime(fieldName, value string) (time.Time, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return time.Time{}, fmt.Errorf("alpaca: %s is required", fieldName) + } + parsed, err := time.Parse(time.RFC3339Nano, trimmed) + if err != nil { + return time.Time{}, fmt.Errorf("alpaca: parse %s: %w", fieldName, err) + } + return parsed.UTC(), nil +} + +func parseOptionalTime(fieldName, value string) (*time.Time, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil, nil + } + parsed, err := time.Parse(time.RFC3339Nano, trimmed) + if err != nil { + return nil, fmt.Errorf("alpaca: parse %s: %w", fieldName, err) + } + parsed = parsed.UTC() + return &parsed, nil +} diff --git a/internal/automation/alpaca_client_adapter_test.go b/internal/automation/alpaca_client_adapter_test.go new file mode 100644 index 00000000..c2b6b270 --- /dev/null +++ b/internal/automation/alpaca_client_adapter_test.go @@ -0,0 +1,208 @@ +package automation + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/PatrickFanella/get-rich-quick/internal/domain" + alpacaexec "github.com/PatrickFanella/get-rich-quick/internal/execution/alpaca" +) + +func TestAlpacaClientAdapterListOrders_MapsBrokerOrders(t *testing.T) { + t.Parallel() + + requests := make(chan *url.URL, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Fatalf("request method = %s, want %s", r.Method, http.MethodGet) + } + if r.URL.Path != "/v2/orders" { + t.Fatalf("request path = %s, want %s", r.URL.Path, "/v2/orders") + } + requests <- r.URL + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[ + { + "id": "order-1", + "symbol": "SNAL", + "side": "buy", + "type": "limit", + "qty": "200", + "limit_price": "0.92", + "filled_qty": "186", + "filled_avg_price": "0.92", + "status": "partially_filled", + "submitted_at": "2026-04-15T19:20:02.451351Z", + "filled_at": "2026-04-15T19:20:04.943982Z" + } + ]`)) + })) + defer server.Close() + + client := alpacaexec.NewClient("test-key", "test-secret", true, slog.New(slog.NewTextHandler(testDiscardWriter{}, nil))) + client.SetBaseURL(server.URL) + + adapter := NewAlpacaClientAdapter(client) + orders, err := adapter.ListOrders(context.Background()) + if err != nil { + t.Fatalf("ListOrders() error = %v", err) + } + if len(orders) != 1 { + t.Fatalf("len(ListOrders()) = %d, want 1", len(orders)) + } + if orders[0].ExternalID != "order-1" { + t.Fatalf("ExternalID = %q, want order-1", orders[0].ExternalID) + } + if orders[0].Ticker != "SNAL" { + t.Fatalf("Ticker = %q, want SNAL", orders[0].Ticker) + } + if orders[0].Side != domain.OrderSideBuy { + t.Fatalf("Side = %q, want buy", orders[0].Side) + } + if orders[0].OrderType != domain.OrderTypeLimit { + t.Fatalf("OrderType = %q, want limit", orders[0].OrderType) + } + if orders[0].Status != domain.OrderStatusPartial { + t.Fatalf("Status = %q, want partial", orders[0].Status) + } + if orders[0].FilledQuantity != 186 { + t.Fatalf("FilledQuantity = %v, want 186", orders[0].FilledQuantity) + } + if orders[0].FilledAvgPrice == nil || *orders[0].FilledAvgPrice != 0.92 { + t.Fatalf("FilledAvgPrice = %v, want 0.92", orders[0].FilledAvgPrice) + } + if orders[0].SubmittedAt == nil || !orders[0].SubmittedAt.Equal(time.Date(2026, 4, 15, 19, 20, 2, 451351000, time.UTC)) { + t.Fatalf("SubmittedAt = %v, want expected timestamp", orders[0].SubmittedAt) + } + if orders[0].FilledAt == nil || !orders[0].FilledAt.Equal(time.Date(2026, 4, 15, 19, 20, 4, 943982000, time.UTC)) { + t.Fatalf("FilledAt = %v, want expected timestamp", orders[0].FilledAt) + } + + select { + case reqURL := <-requests: + if got := reqURL.Query().Get("status"); got != "all" { + t.Fatalf("status query = %q, want all", got) + } + if got := reqURL.Query().Get("limit"); got != "500" { + t.Fatalf("limit query = %q, want 500", got) + } + if got := reqURL.Query().Get("direction"); got != "desc" { + t.Fatalf("direction query = %q, want desc", got) + } + case <-time.After(time.Second): + t.Fatal("request details were not captured") + } +} + +func TestAlpacaClientAdapterListFills_PaginatesActivities(t *testing.T) { + t.Parallel() + + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Fatalf("request method = %s, want %s", r.Method, http.MethodGet) + } + if r.URL.Path != "/v2/account/activities/FILL" { + t.Fatalf("request path = %s, want %s", r.URL.Path, "/v2/account/activities/FILL") + } + requestCount++ + w.Header().Set("Content-Type", "application/json") + switch requestCount { + case 1: + if got := r.URL.Query().Get("page_token"); got != "" { + t.Fatalf("first page_token = %q, want empty", got) + } + for i := 0; i < alpacaActivitiesPageSize; i++ { + activityID := "fill-1" + qty := "186" + transactionTime := "2026-04-15T19:20:02.66268Z" + orderStatus := "partially_filled" + if i == 0 { + activityID = "fill-2" + qty = "14" + transactionTime = "2026-04-15T19:20:04.943982Z" + orderStatus = "filled" + } + if i > 1 { + activityID = activityID + "-extra" + } + if i == 0 { + _, _ = w.Write([]byte("[")) + } else { + _, _ = w.Write([]byte(",")) + } + _, _ = w.Write([]byte(`{ + "activity_type": "FILL", + "id": "` + activityID + `", + "order_id": "order-1", + "symbol": "SNAL", + "side": "buy", + "qty": "` + qty + `", + "price": "0.92", + "transaction_time": "` + transactionTime + `", + "order_status": "` + orderStatus + `" + }`)) + } + _, _ = w.Write([]byte("]")) + case 2: + if got := r.URL.Query().Get("page_token"); got != "fill-1-extra" { + t.Fatalf("second page_token = %q, want fill-1-extra", got) + } + _, _ = w.Write([]byte(`[ + { + "activity_type": "FILL", + "id": "fill-3", + "order_id": "order-1", + "symbol": "SNAL", + "side": "buy", + "qty": "1", + "price": "0.93", + "transaction_time": "2026-04-15T19:21:04.943982Z", + "order_status": "filled" + } + ]`)) + default: + t.Fatalf("unexpected request count %d", requestCount) + } + })) + defer server.Close() + + client := alpacaexec.NewClient("test-key", "test-secret", true, slog.New(slog.NewTextHandler(testDiscardWriter{}, nil))) + client.SetBaseURL(server.URL) + + adapter := NewAlpacaClientAdapter(client) + fills, err := adapter.ListFills(context.Background()) + if err != nil { + t.Fatalf("ListFills() error = %v", err) + } + if requestCount != 2 { + t.Fatalf("requestCount = %d, want 2", requestCount) + } + if len(fills) != alpacaActivitiesPageSize+1 { + t.Fatalf("len(ListFills()) = %d, want %d", len(fills), alpacaActivitiesPageSize+1) + } + if fills[0].ActivityID != "fill-2" { + t.Fatalf("fills[0].ActivityID = %q, want fill-2", fills[0].ActivityID) + } + if fills[0].OrderStatus != domain.OrderStatusFilled { + t.Fatalf("fills[0].OrderStatus = %q, want filled", fills[0].OrderStatus) + } + if fills[1].ActivityID != "fill-1" { + t.Fatalf("fills[1].ActivityID = %q, want fill-1", fills[1].ActivityID) + } + if fills[1].OrderStatus != domain.OrderStatusPartial { + t.Fatalf("fills[1].OrderStatus = %q, want partial", fills[1].OrderStatus) + } + if fills[len(fills)-1].ActivityID != "fill-3" { + t.Fatalf("last fill ActivityID = %q, want fill-3", fills[len(fills)-1].ActivityID) + } +} + +type testDiscardWriter struct{} + +func (testDiscardWriter) Write(p []byte) (int, error) { return len(p), nil } diff --git a/internal/automation/alpaca_reconciliation.go b/internal/automation/alpaca_reconciliation.go new file mode 100644 index 00000000..eb517359 --- /dev/null +++ b/internal/automation/alpaca_reconciliation.go @@ -0,0 +1,790 @@ +package automation + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "sort" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/PatrickFanella/get-rich-quick/internal/domain" + "github.com/PatrickFanella/get-rich-quick/internal/repository" +) + +// AlpacaReconciliationBroker fetches current broker snapshots needed for local reconciliation. +type AlpacaReconciliationBroker interface { + GetPositions(ctx context.Context) ([]domain.Position, error) + ListOrders(ctx context.Context) ([]BrokerOrderSnapshot, error) + ListFills(ctx context.Context) ([]BrokerFillSnapshot, error) +} + +// StrategyLookupRepository is the narrow strategy dependency needed by reconciliation. +type StrategyLookupRepository interface { + List(ctx context.Context, filter repository.StrategyFilter, limit, offset int) ([]domain.Strategy, error) +} + +// OrderPersistence is the narrow order repository surface needed by reconciliation. +type OrderPersistence interface { + Create(ctx context.Context, order *domain.Order) error + List(ctx context.Context, filter repository.OrderFilter, limit, offset int) ([]domain.Order, error) + Update(ctx context.Context, order *domain.Order) error +} + +// PositionPersistence is the narrow position repository surface needed by reconciliation. +type PositionPersistence interface { + Create(ctx context.Context, position *domain.Position) error + GetOpen(ctx context.Context, filter repository.PositionFilter, limit, offset int) ([]domain.Position, error) + Update(ctx context.Context, position *domain.Position) error +} + +// TradePersistence is the narrow trade repository surface needed by reconciliation. +type TradePersistence interface { + Create(ctx context.Context, trade *domain.Trade) error + List(ctx context.Context, filter repository.TradeFilter, limit, offset int) ([]domain.Trade, error) +} + +// BrokerOrderSnapshot captures the broker-facing order state needed to hydrate local orders. +type BrokerOrderSnapshot struct { + ExternalID string + StrategyIDHint *uuid.UUID + Ticker string + Side domain.OrderSide + OrderType domain.OrderType + Quantity float64 + LimitPrice *float64 + StopPrice *float64 + FilledQuantity float64 + FilledAvgPrice *float64 + Status domain.OrderStatus + SubmittedAt *time.Time + FilledAt *time.Time + Broker string +} + +// BrokerFillSnapshot captures a single broker fill activity. +type BrokerFillSnapshot struct { + ActivityID string + ExternalID string + Ticker string + Side domain.OrderSide + Quantity float64 + Price float64 + Fee float64 + ExecutedAt time.Time + OrderStatus domain.OrderStatus +} + +// AlpacaReconcilerDeps bundles repository and broker dependencies. +type AlpacaReconcilerDeps struct { + Broker AlpacaReconciliationBroker + StrategyRepo StrategyLookupRepository + OrderRepo OrderPersistence + PositionRepo PositionPersistence + TradeRepo TradePersistence + AuditLogRepo repository.AuditLogRepository + Logger *slog.Logger +} + +// AlpacaReconcileSummary reports how many local records changed during a run. +type AlpacaReconcileSummary struct { + OrdersCreated int + OrdersUpdated int + PositionsCreated int + PositionsUpdated int + TradesCreated int +} + +type AlpacaVerificationMismatch struct { + Entity string `json:"entity"` + Key string `json:"key"` + Details []string `json:"details,omitempty"` +} + +type AlpacaVerificationReport struct { + OrdersChecked int `json:"orders_checked"` + PositionsChecked int `json:"positions_checked"` + FillsChecked int `json:"fills_checked"` + MissingOrders int `json:"missing_orders"` + MissingPositions int `json:"missing_positions"` + MissingTrades int `json:"missing_trades"` + Mismatches []AlpacaVerificationMismatch `json:"mismatches,omitempty"` + Verified bool `json:"verified"` +} + +func (s AlpacaReconcileSummary) Map() map[string]int { + return map[string]int{ + "orders_created": s.OrdersCreated, + "orders_updated": s.OrdersUpdated, + "positions_created": s.PositionsCreated, + "positions_updated": s.PositionsUpdated, + "trades_created": s.TradesCreated, + } +} + +// AlpacaReconciler imports Alpaca broker state into local orders, positions, and trades tables. +type AlpacaReconciler struct { + broker AlpacaReconciliationBroker + strategyRepo StrategyLookupRepository + orderRepo OrderPersistence + positionRepo PositionPersistence + tradeRepo TradePersistence + auditLogRepo repository.AuditLogRepository + logger *slog.Logger +} + +func NewAlpacaReconciler(deps AlpacaReconcilerDeps) *AlpacaReconciler { + logger := deps.Logger + if logger == nil { + logger = slog.Default() + } + return &AlpacaReconciler{ + broker: deps.Broker, + strategyRepo: deps.StrategyRepo, + orderRepo: deps.OrderRepo, + positionRepo: deps.PositionRepo, + tradeRepo: deps.TradeRepo, + auditLogRepo: deps.AuditLogRepo, + logger: logger, + } +} + +func (r *AlpacaReconciler) Reconcile(ctx context.Context) (AlpacaReconcileSummary, error) { + if r == nil || r.broker == nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: broker is required") + } + if r.orderRepo == nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: order repository is required") + } + if r.positionRepo == nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: position repository is required") + } + if r.tradeRepo == nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: trade repository is required") + } + + positions, err := r.broker.GetPositions(ctx) + if err != nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: fetch positions: %w", err) + } + orders, err := r.broker.ListOrders(ctx) + if err != nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: fetch orders: %w", err) + } + fills, err := r.broker.ListFills(ctx) + if err != nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: fetch fills: %w", err) + } + + strategyByTicker, err := r.loadStrategyIndex(ctx) + if err != nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: load strategy index: %w", err) + } + + existingOrders, err := r.orderRepo.List(ctx, repository.OrderFilter{Broker: "alpaca"}, 1000, 0) + if err != nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: list local orders: %w", err) + } + orderByExternalID := make(map[string]*domain.Order, len(existingOrders)) + for i := range existingOrders { + order := existingOrders[i] + if strings.TrimSpace(order.ExternalID) == "" { + continue + } + cloned := order + orderByExternalID[order.ExternalID] = &cloned + } + + existingPositions, err := r.positionRepo.GetOpen(ctx, repository.PositionFilter{}, 1000, 0) + if err != nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: list local positions: %w", err) + } + positionByTicker := make(map[string]*domain.Position, len(existingPositions)) + for i := range existingPositions { + position := existingPositions[i] + cloned := position + positionByTicker[position.Ticker] = &cloned + } + + existingTrades, err := r.tradeRepo.List(ctx, repository.TradeFilter{}, 5000, 0) + if err != nil { + return AlpacaReconcileSummary{}, fmt.Errorf("alpaca_reconcile: list local trades: %w", err) + } + existingTradeKeys := make(map[string]struct{}, len(existingTrades)) + for _, trade := range existingTrades { + existingTradeKeys[tradeDedupeKey(trade)] = struct{}{} + } + fillLegacyKeyCounts := fillLegacyKeyCounts(fills) + + summary := AlpacaReconcileSummary{} + + for _, snapshot := range orders { + strategyID := snapshot.StrategyIDHint + if strategyID == nil { + strategyID = strategyByTicker[snapshot.Ticker] + } + if existing, ok := orderByExternalID[snapshot.ExternalID]; ok { + changed := applyOrderSnapshot(existing, snapshot, strategyID) + if changed { + if err := r.orderRepo.Update(ctx, existing); err != nil { + return summary, fmt.Errorf("alpaca_reconcile: update order %s: %w", snapshot.ExternalID, err) + } + summary.OrdersUpdated++ + } + continue + } + + order := snapshotToOrder(snapshot, strategyID) + if err := r.orderRepo.Create(ctx, order); err != nil { + return summary, fmt.Errorf("alpaca_reconcile: create order %s: %w", snapshot.ExternalID, err) + } + orderByExternalID[snapshot.ExternalID] = order + summary.OrdersCreated++ + } + + for _, snapshot := range positions { + strategyID := strategyByTicker[snapshot.Ticker] + if existing, ok := positionByTicker[snapshot.Ticker]; ok { + changed := applyPositionSnapshot(existing, snapshot, strategyID) + if changed { + if err := r.positionRepo.Update(ctx, existing); err != nil { + return summary, fmt.Errorf("alpaca_reconcile: update position %s: %w", snapshot.Ticker, err) + } + summary.PositionsUpdated++ + } + continue + } + + position := snapshotToPosition(snapshot, strategyID) + if err := r.positionRepo.Create(ctx, position); err != nil { + return summary, fmt.Errorf("alpaca_reconcile: create position %s: %w", snapshot.Ticker, err) + } + positionByTicker[position.Ticker] = position + summary.PositionsCreated++ + } + + sort.Slice(fills, func(i, j int) bool { + return fills[i].ExecutedAt.Before(fills[j].ExecutedAt) + }) + for _, fill := range fills { + fillKeys := dedupeKeysForFill(fill, fillLegacyKeyCounts) + skip := false + for _, key := range fillKeys { + if _, ok := existingTradeKeys[key]; ok { + skip = true + break + } + } + if skip { + continue + } + + order := orderByExternalID[fill.ExternalID] + if order == nil { + continue + } + position := positionByTicker[fill.Ticker] + trade := &domain.Trade{ + OrderID: &order.ID, + PositionID: nil, + ExternalID: strings.TrimSpace(fill.ActivityID), + Ticker: fill.Ticker, + Side: fill.Side, + Quantity: fill.Quantity, + Price: fill.Price, + Fee: fill.Fee, + ExecutedAt: fill.ExecutedAt, + } + if position != nil { + trade.PositionID = &position.ID + } + if err := r.tradeRepo.Create(ctx, trade); err != nil { + return summary, fmt.Errorf("alpaca_reconcile: create trade for %s: %w", fill.ExternalID, err) + } + for _, key := range fillKeys { + existingTradeKeys[key] = struct{}{} + } + summary.TradesCreated++ + } + + if err := r.recordAudit(ctx, "alpaca_reconcile.completed", summary.Map()); err != nil { + r.logger.Warn("alpaca_reconcile: failed to record audit entry", slog.Any("error", err)) + } + + return summary, nil +} + +func (r *AlpacaReconciler) Verify(ctx context.Context) (AlpacaVerificationReport, error) { + if r == nil || r.broker == nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: broker is required") + } + if r.orderRepo == nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: order repository is required") + } + if r.positionRepo == nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: position repository is required") + } + if r.tradeRepo == nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: trade repository is required") + } + + positions, err := r.broker.GetPositions(ctx) + if err != nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: fetch positions: %w", err) + } + orders, err := r.broker.ListOrders(ctx) + if err != nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: fetch orders: %w", err) + } + fills, err := r.broker.ListFills(ctx) + if err != nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: fetch fills: %w", err) + } + + localOrders, err := r.orderRepo.List(ctx, repository.OrderFilter{Broker: "alpaca"}, 1000, 0) + if err != nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: list local orders: %w", err) + } + localPositions, err := r.positionRepo.GetOpen(ctx, repository.PositionFilter{}, 1000, 0) + if err != nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: list local positions: %w", err) + } + localTrades, err := r.tradeRepo.List(ctx, repository.TradeFilter{}, 5000, 0) + if err != nil { + return AlpacaVerificationReport{}, fmt.Errorf("alpaca_reconcile: list local trades: %w", err) + } + + orderByExternalID := make(map[string]domain.Order, len(localOrders)) + for _, order := range localOrders { + if strings.TrimSpace(order.ExternalID) == "" { + continue + } + orderByExternalID[order.ExternalID] = order + } + positionByTicker := make(map[string]domain.Position, len(localPositions)) + for _, position := range localPositions { + positionByTicker[position.Ticker] = position + } + tradeByKey := make(map[string]domain.Trade, len(localTrades)) + for _, trade := range localTrades { + tradeByKey[tradeDedupeKey(trade)] = trade + } + fillLegacyKeyCounts := fillLegacyKeyCounts(fills) + + report := AlpacaVerificationReport{ + OrdersChecked: len(orders), + PositionsChecked: len(positions), + FillsChecked: len(fills), + Verified: true, + } + + for _, snapshot := range orders { + localOrder, ok := orderByExternalID[snapshot.ExternalID] + if !ok { + report.MissingOrders++ + report.Mismatches = append(report.Mismatches, AlpacaVerificationMismatch{Entity: "order", Key: snapshot.ExternalID, Details: []string{"missing local order"}}) + continue + } + if fields := diffOrderSnapshot(localOrder, snapshot); len(fields) > 0 { + report.Mismatches = append(report.Mismatches, AlpacaVerificationMismatch{Entity: "order", Key: snapshot.ExternalID, Details: fields}) + } + } + + for _, snapshot := range positions { + localPosition, ok := positionByTicker[snapshot.Ticker] + if !ok { + report.MissingPositions++ + report.Mismatches = append(report.Mismatches, AlpacaVerificationMismatch{Entity: "position", Key: snapshot.Ticker, Details: []string{"missing local position"}}) + continue + } + if fields := diffPositionSnapshot(localPosition, snapshot); len(fields) > 0 { + report.Mismatches = append(report.Mismatches, AlpacaVerificationMismatch{Entity: "position", Key: snapshot.Ticker, Details: fields}) + } + } + + for _, fill := range fills { + fillKeys := dedupeKeysForFill(fill, fillLegacyKeyCounts) + localTrade, ok := tradeByKey[fillKeys[0]] + if !ok { + for _, key := range fillKeys[1:] { + if localTrade, ok = tradeByKey[key]; ok { + break + } + } + } + if !ok { + report.MissingTrades++ + report.Mismatches = append(report.Mismatches, AlpacaVerificationMismatch{Entity: "trade", Key: fillKeys[0], Details: []string{"missing local trade"}}) + continue + } + if fields := diffTradeFill(localTrade, fill); len(fields) > 0 { + report.Mismatches = append(report.Mismatches, AlpacaVerificationMismatch{Entity: "trade", Key: fillKeys[0], Details: fields}) + } + } + + report.Verified = report.MissingOrders == 0 && report.MissingPositions == 0 && report.MissingTrades == 0 && len(report.Mismatches) == 0 + if err := r.recordAudit(ctx, "alpaca_reconcile.verified", map[string]any{ + "orders_checked": report.OrdersChecked, + "positions_checked": report.PositionsChecked, + "fills_checked": report.FillsChecked, + "missing_orders": report.MissingOrders, + "missing_positions": report.MissingPositions, + "missing_trades": report.MissingTrades, + "verified": report.Verified, + "mismatches": report.Mismatches, + }); err != nil { + r.logger.Warn("alpaca_reconcile: failed to record verification audit entry", slog.Any("error", err)) + } + return report, nil +} + +func (r *AlpacaReconciler) recordAudit(ctx context.Context, eventType string, details any) error { + if r.auditLogRepo == nil { + return nil + } + payload, err := json.Marshal(details) + if err != nil { + return fmt.Errorf("marshal audit details: %w", err) + } + return r.auditLogRepo.Create(ctx, &domain.AuditLogEntry{ + EventType: eventType, + EntityType: "automation_job", + Actor: "alpaca_reconciler", + Details: payload, + }) +} + +func (r *AlpacaReconciler) loadStrategyIndex(ctx context.Context) (map[string]*uuid.UUID, error) { + if r.strategyRepo == nil { + return map[string]*uuid.UUID{}, nil + } + strategies, err := r.strategyRepo.List(ctx, repository.StrategyFilter{Status: domain.StrategyStatusActive}, 1000, 0) + if err != nil { + return nil, err + } + result := make(map[string]*uuid.UUID, len(strategies)) + for _, strategy := range strategies { + id := strategy.ID + result[strategy.Ticker] = &id + } + return result, nil +} + +func snapshotToOrder(snapshot BrokerOrderSnapshot, strategyID *uuid.UUID) *domain.Order { + return &domain.Order{ + StrategyID: strategyID, + ExternalID: snapshot.ExternalID, + Ticker: snapshot.Ticker, + Side: snapshot.Side, + OrderType: snapshot.OrderType, + Quantity: snapshot.Quantity, + LimitPrice: cloneFloatPtr(snapshot.LimitPrice), + StopPrice: cloneFloatPtr(snapshot.StopPrice), + FilledQuantity: snapshot.FilledQuantity, + FilledAvgPrice: cloneFloatPtr(snapshot.FilledAvgPrice), + Status: snapshot.Status, + Broker: fallbackBroker(snapshot.Broker), + SubmittedAt: cloneTimePtr(snapshot.SubmittedAt), + FilledAt: cloneTimePtr(snapshot.FilledAt), + } +} + +func snapshotToPosition(snapshot domain.Position, strategyID *uuid.UUID) *domain.Position { + return &domain.Position{ + StrategyID: strategyID, + Ticker: snapshot.Ticker, + Side: snapshot.Side, + Quantity: snapshot.Quantity, + AvgEntry: snapshot.AvgEntry, + CurrentPrice: cloneFloatPtr(snapshot.CurrentPrice), + UnrealizedPnL: cloneFloatPtr(snapshot.UnrealizedPnL), + } +} + +func applyOrderSnapshot(order *domain.Order, snapshot BrokerOrderSnapshot, strategyID *uuid.UUID) bool { + changed := false + if !uuidPtrEqual(order.StrategyID, strategyID) { + order.StrategyID = cloneUUIDPtr(strategyID) + changed = true + } + if order.Ticker != snapshot.Ticker { + order.Ticker = snapshot.Ticker + changed = true + } + if order.Side != snapshot.Side { + order.Side = snapshot.Side + changed = true + } + if order.OrderType != snapshot.OrderType { + order.OrderType = snapshot.OrderType + changed = true + } + if order.Quantity != snapshot.Quantity { + order.Quantity = snapshot.Quantity + changed = true + } + if !floatPtrEqual(order.LimitPrice, snapshot.LimitPrice) { + order.LimitPrice = cloneFloatPtr(snapshot.LimitPrice) + changed = true + } + if !floatPtrEqual(order.StopPrice, snapshot.StopPrice) { + order.StopPrice = cloneFloatPtr(snapshot.StopPrice) + changed = true + } + if order.FilledQuantity != snapshot.FilledQuantity { + order.FilledQuantity = snapshot.FilledQuantity + changed = true + } + if !floatPtrEqual(order.FilledAvgPrice, snapshot.FilledAvgPrice) { + order.FilledAvgPrice = cloneFloatPtr(snapshot.FilledAvgPrice) + changed = true + } + if order.Status != snapshot.Status { + order.Status = snapshot.Status + changed = true + } + broker := fallbackBroker(snapshot.Broker) + if order.Broker != broker { + order.Broker = broker + changed = true + } + if !timePtrEqual(order.SubmittedAt, snapshot.SubmittedAt) { + order.SubmittedAt = cloneTimePtr(snapshot.SubmittedAt) + changed = true + } + if !timePtrEqual(order.FilledAt, snapshot.FilledAt) { + order.FilledAt = cloneTimePtr(snapshot.FilledAt) + changed = true + } + return changed +} + +func applyPositionSnapshot(position *domain.Position, snapshot domain.Position, strategyID *uuid.UUID) bool { + changed := false + if !uuidPtrEqual(position.StrategyID, strategyID) { + position.StrategyID = cloneUUIDPtr(strategyID) + changed = true + } + if position.Side != snapshot.Side { + position.Side = snapshot.Side + changed = true + } + if position.Quantity != snapshot.Quantity { + position.Quantity = snapshot.Quantity + changed = true + } + if position.AvgEntry != snapshot.AvgEntry { + position.AvgEntry = snapshot.AvgEntry + changed = true + } + if !floatPtrEqual(position.CurrentPrice, snapshot.CurrentPrice) { + position.CurrentPrice = cloneFloatPtr(snapshot.CurrentPrice) + changed = true + } + if !floatPtrEqual(position.UnrealizedPnL, snapshot.UnrealizedPnL) { + position.UnrealizedPnL = cloneFloatPtr(snapshot.UnrealizedPnL) + changed = true + } + return changed +} + +func fillDedupeKey(fill BrokerFillSnapshot) string { + if activityID := strings.TrimSpace(fill.ActivityID); activityID != "" { + return strings.Join([]string{"activity", activityID}, "|") + } + return tradeExecutionDedupeKey(fill.Ticker, fill.Side, fill.Quantity, fill.Price, fill.ExecutedAt) +} + +func fillLegacyKey(fill BrokerFillSnapshot) string { + return tradeExecutionDedupeKey(fill.Ticker, fill.Side, fill.Quantity, fill.Price, fill.ExecutedAt) +} + +func fillLegacyKeyCounts(fills []BrokerFillSnapshot) map[string]int { + counts := make(map[string]int, len(fills)) + for _, fill := range fills { + counts[fillLegacyKey(fill)]++ + } + return counts +} + +func dedupeKeysForFill(fill BrokerFillSnapshot, legacyCounts map[string]int) []string { + primary := fillDedupeKey(fill) + legacy := fillLegacyKey(fill) + if legacy == primary || legacyCounts[legacy] > 1 { + return []string{primary} + } + return []string{primary, legacy} +} + +func tradeDedupeKey(trade domain.Trade) string { + if externalID := strings.TrimSpace(trade.ExternalID); externalID != "" { + return strings.Join([]string{"activity", externalID}, "|") + } + return legacyTradeDedupeKey(trade) +} + +func legacyTradeDedupeKey(trade domain.Trade) string { + return tradeExecutionDedupeKey(trade.Ticker, trade.Side, trade.Quantity, trade.Price, trade.ExecutedAt) +} + +func tradeExecutionDedupeKey(ticker string, side domain.OrderSide, quantity, price float64, executedAt time.Time) string { + return strings.Join([]string{ + ticker, + side.String(), + formatFloat(quantity), + formatFloat(price), + executedAt.UTC().Format(time.RFC3339Nano), + }, "|") +} + +func fallbackBroker(broker string) string { + trimmed := strings.TrimSpace(broker) + if trimmed == "" { + return "alpaca" + } + return trimmed +} + +func diffOrderSnapshot(order domain.Order, snapshot BrokerOrderSnapshot) []string { + var fields []string + if order.Ticker != snapshot.Ticker { + fields = append(fields, "ticker") + } + if order.Side != snapshot.Side { + fields = append(fields, "side") + } + if order.OrderType != snapshot.OrderType { + fields = append(fields, "order_type") + } + if !normalizedQuantityEqual(order.Quantity, snapshot.Quantity) { + fields = append(fields, "quantity") + } + if !floatPtrEqual(order.LimitPrice, snapshot.LimitPrice) { + fields = append(fields, "limit_price") + } + if !floatPtrEqual(order.StopPrice, snapshot.StopPrice) { + fields = append(fields, "stop_price") + } + if !normalizedQuantityEqual(order.FilledQuantity, snapshot.FilledQuantity) { + fields = append(fields, "filled_quantity") + } + if !floatPtrEqual(order.FilledAvgPrice, snapshot.FilledAvgPrice) { + fields = append(fields, "filled_avg_price") + } + if order.Status != snapshot.Status { + fields = append(fields, "status") + } + if order.Broker != fallbackBroker(snapshot.Broker) { + fields = append(fields, "broker") + } + if !timePtrEqual(order.SubmittedAt, snapshot.SubmittedAt) { + fields = append(fields, "submitted_at") + } + if !timePtrEqual(order.FilledAt, snapshot.FilledAt) { + fields = append(fields, "filled_at") + } + return fields +} + +func diffPositionSnapshot(position domain.Position, snapshot domain.Position) []string { + var fields []string + if position.Ticker != snapshot.Ticker { + fields = append(fields, "ticker") + } + if position.Side != snapshot.Side { + fields = append(fields, "side") + } + if position.Quantity != snapshot.Quantity { + fields = append(fields, "quantity") + } + if position.AvgEntry != snapshot.AvgEntry { + fields = append(fields, "avg_entry") + } + return fields +} + +func diffTradeFill(trade domain.Trade, fill BrokerFillSnapshot) []string { + var fields []string + if activityID := strings.TrimSpace(fill.ActivityID); activityID != "" && strings.TrimSpace(trade.ExternalID) != activityID { + fields = append(fields, "external_id") + } + if trade.Ticker != fill.Ticker { + fields = append(fields, "ticker") + } + if trade.Side != fill.Side { + fields = append(fields, "side") + } + if trade.Quantity != fill.Quantity { + fields = append(fields, "quantity") + } + if trade.Price != fill.Price { + fields = append(fields, "price") + } + if trade.Fee != fill.Fee { + fields = append(fields, "fee") + } + if !trade.ExecutedAt.Equal(fill.ExecutedAt) { + fields = append(fields, "executed_at") + } + return fields +} + +func cloneFloatPtr(value *float64) *float64 { + if value == nil { + return nil + } + v := *value + return &v +} + +func cloneTimePtr(value *time.Time) *time.Time { + if value == nil { + return nil + } + v := *value + return &v +} + +func cloneUUIDPtr(value *uuid.UUID) *uuid.UUID { + if value == nil { + return nil + } + v := *value + return &v +} + +func floatPtrEqual(left, right *float64) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return *left == *right +} + +func timePtrEqual(left, right *time.Time) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return left.Equal(*right) +} + +func uuidPtrEqual(left, right *uuid.UUID) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return *left == *right +} + +func formatFloat(v float64) string { + return fmt.Sprintf("%.10f", v) +} + +func normalizedQuantityEqual(left, right float64) bool { + return formatStorageNumeric(left) == formatStorageNumeric(right) +} + +func formatStorageNumeric(v float64) string { + return fmt.Sprintf("%.8f", v) +} diff --git a/internal/automation/alpaca_reconciliation_test.go b/internal/automation/alpaca_reconciliation_test.go new file mode 100644 index 00000000..bc34aef0 --- /dev/null +++ b/internal/automation/alpaca_reconciliation_test.go @@ -0,0 +1,1041 @@ +package automation + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/PatrickFanella/get-rich-quick/internal/domain" + "github.com/PatrickFanella/get-rich-quick/internal/repository" +) + +type alpacaReconciliationBrokerStub struct { + positions []domain.Position + orders []BrokerOrderSnapshot + fills []BrokerFillSnapshot + + positionsErr error + ordersErr error + fillsErr error +} + +func (s *alpacaReconciliationBrokerStub) GetPositions(ctx context.Context) ([]domain.Position, error) { + if s.positionsErr != nil { + return nil, s.positionsErr + } + out := make([]domain.Position, len(s.positions)) + copy(out, s.positions) + return out, nil +} + +func (s *alpacaReconciliationBrokerStub) ListOrders(ctx context.Context) ([]BrokerOrderSnapshot, error) { + if s.ordersErr != nil { + return nil, s.ordersErr + } + out := make([]BrokerOrderSnapshot, len(s.orders)) + copy(out, s.orders) + return out, nil +} + +func (s *alpacaReconciliationBrokerStub) ListFills(ctx context.Context) ([]BrokerFillSnapshot, error) { + if s.fillsErr != nil { + return nil, s.fillsErr + } + out := make([]BrokerFillSnapshot, len(s.fills)) + copy(out, s.fills) + return out, nil +} + +type recordingStrategyRepo struct { + list []domain.Strategy + err error +} + +func (r *recordingStrategyRepo) Create(context.Context, *domain.Strategy) error { return nil } +func (r *recordingStrategyRepo) Get(context.Context, uuid.UUID) (*domain.Strategy, error) { + return nil, repository.ErrNotFound +} +func (r *recordingStrategyRepo) List(context.Context, repository.StrategyFilter, int, int) ([]domain.Strategy, error) { + if r.err != nil { + return nil, r.err + } + out := make([]domain.Strategy, len(r.list)) + copy(out, r.list) + return out, nil +} +func (r *recordingStrategyRepo) Count(context.Context, repository.StrategyFilter) (int, error) { + return len(r.list), nil +} +func (r *recordingStrategyRepo) Update(context.Context, *domain.Strategy) error { return nil } +func (r *recordingStrategyRepo) Delete(context.Context, uuid.UUID) error { return nil } +func (r *recordingStrategyRepo) ValidateConfig(context.Context, domain.MarketType, []byte) error { + return nil +} +func (r *recordingStrategyRepo) GetByTicker(context.Context, string, repository.StrategyFilter, int, int) ([]domain.Strategy, error) { + return nil, nil +} +func (r *recordingStrategyRepo) CountByTicker(context.Context, string, repository.StrategyFilter) (int, error) { + return 0, nil +} +func (r *recordingStrategyRepo) UpdateThesis(context.Context, uuid.UUID, json.RawMessage) error { + return nil +} +func (r *recordingStrategyRepo) GetThesisRaw(context.Context, uuid.UUID) (json.RawMessage, error) { + return nil, nil +} + +type recordingOrderRepo struct { + byExternalID map[string]*domain.Order + created []*domain.Order + updated []*domain.Order + list []*domain.Order +} + +func newRecordingOrderRepo(existing ...*domain.Order) *recordingOrderRepo { + idx := make(map[string]*domain.Order) + list := make([]*domain.Order, 0, len(existing)) + for _, order := range existing { + cloned := cloneOrder(order) + if cloned.ExternalID != "" { + idx[cloned.ExternalID] = cloned + } + list = append(list, cloned) + } + return &recordingOrderRepo{byExternalID: idx, list: list} +} + +func (r *recordingOrderRepo) Create(_ context.Context, order *domain.Order) error { + cloned := cloneOrder(order) + if cloned.ID == uuid.Nil { + cloned.ID = uuid.New() + } + if cloned.CreatedAt.IsZero() { + cloned.CreatedAt = time.Now().UTC() + } + *order = *cloned + r.created = append(r.created, cloneOrder(cloned)) + r.list = append(r.list, cloned) + if cloned.ExternalID != "" { + r.byExternalID[cloned.ExternalID] = cloned + } + return nil +} + +func (r *recordingOrderRepo) Get(_ context.Context, id uuid.UUID) (*domain.Order, error) { + for _, order := range r.list { + if order.ID == id { + return cloneOrder(order), nil + } + } + return nil, repository.ErrNotFound +} + +func (r *recordingOrderRepo) List(_ context.Context, filter repository.OrderFilter, limit, offset int) ([]domain.Order, error) { + var filtered []domain.Order + for _, order := range r.list { + if filter.Broker != "" && order.Broker != filter.Broker { + continue + } + if filter.Ticker != "" && order.Ticker != filter.Ticker { + continue + } + filtered = append(filtered, *cloneOrder(order)) + } + return paginateOrders(filtered, limit, offset), nil +} + +func (r *recordingOrderRepo) Count(ctx context.Context, filter repository.OrderFilter) (int, error) { + orders, err := r.List(ctx, filter, 0, 0) + if err != nil { + return 0, err + } + return len(orders), nil +} + +func (r *recordingOrderRepo) Update(_ context.Context, order *domain.Order) error { + for i, existing := range r.list { + if existing.ID == order.ID { + cloned := cloneOrder(order) + r.list[i] = cloned + if cloned.ExternalID != "" { + r.byExternalID[cloned.ExternalID] = cloned + } + r.updated = append(r.updated, cloneOrder(cloned)) + return nil + } + } + return repository.ErrNotFound +} + +func (r *recordingOrderRepo) Delete(_ context.Context, id uuid.UUID) error { return nil } +func (r *recordingOrderRepo) GetByStrategy(_ context.Context, _ uuid.UUID, _ repository.OrderFilter, _, _ int) ([]domain.Order, error) { + return nil, nil +} +func (r *recordingOrderRepo) GetByRun(_ context.Context, _ uuid.UUID, _ repository.OrderFilter, _, _ int) ([]domain.Order, error) { + return nil, nil +} + +type recordingPositionRepo struct { + open []*domain.Position + created []*domain.Position + updated []*domain.Position +} + +func newRecordingPositionRepo(existing ...*domain.Position) *recordingPositionRepo { + open := make([]*domain.Position, 0, len(existing)) + for _, position := range existing { + open = append(open, clonePosition(position)) + } + return &recordingPositionRepo{open: open} +} + +func (r *recordingPositionRepo) Create(_ context.Context, position *domain.Position) error { + cloned := clonePosition(position) + if cloned.ID == uuid.Nil { + cloned.ID = uuid.New() + } + if cloned.OpenedAt.IsZero() { + cloned.OpenedAt = time.Now().UTC() + } + *position = *cloned + r.created = append(r.created, clonePosition(cloned)) + r.open = append(r.open, cloned) + return nil +} + +func (r *recordingPositionRepo) Get(_ context.Context, id uuid.UUID) (*domain.Position, error) { + for _, position := range r.open { + if position.ID == id { + return clonePosition(position), nil + } + } + return nil, repository.ErrNotFound +} + +func (r *recordingPositionRepo) List(_ context.Context, filter repository.PositionFilter, limit, offset int) ([]domain.Position, error) { + var filtered []domain.Position + for _, position := range r.open { + if filter.Ticker != "" && position.Ticker != filter.Ticker { + continue + } + filtered = append(filtered, *clonePosition(position)) + } + return paginatePositions(filtered, limit, offset), nil +} + +func (r *recordingPositionRepo) Count(ctx context.Context, filter repository.PositionFilter) (int, error) { + positions, err := r.List(ctx, filter, 0, 0) + if err != nil { + return 0, err + } + return len(positions), nil +} + +func (r *recordingPositionRepo) Update(_ context.Context, position *domain.Position) error { + for i, existing := range r.open { + if existing.ID == position.ID { + cloned := clonePosition(position) + r.open[i] = cloned + r.updated = append(r.updated, clonePosition(cloned)) + return nil + } + } + return repository.ErrNotFound +} + +func (r *recordingPositionRepo) Delete(_ context.Context, _ uuid.UUID) error { return nil } + +func (r *recordingPositionRepo) GetOpen(_ context.Context, filter repository.PositionFilter, limit, offset int) ([]domain.Position, error) { + var filtered []domain.Position + for _, position := range r.open { + if position.ClosedAt != nil { + continue + } + if filter.Ticker != "" && position.Ticker != filter.Ticker { + continue + } + filtered = append(filtered, *clonePosition(position)) + } + return paginatePositions(filtered, limit, offset), nil +} + +func (r *recordingPositionRepo) CountOpen(ctx context.Context, filter repository.PositionFilter) (int, error) { + positions, err := r.GetOpen(ctx, filter, 0, 0) + if err != nil { + return 0, err + } + return len(positions), nil +} +func (r *recordingPositionRepo) GetByStrategy(_ context.Context, _ uuid.UUID, _ repository.PositionFilter, _, _ int) ([]domain.Position, error) { + return nil, nil +} + +type recordingTradeRepo struct { + created []*domain.Trade + byOrderExternalID map[string][]*domain.Trade + orders *recordingOrderRepo +} + +func newRecordingTradeRepo(orders *recordingOrderRepo) *recordingTradeRepo { + return &recordingTradeRepo{orders: orders, byOrderExternalID: map[string][]*domain.Trade{}} +} + +func (r *recordingTradeRepo) seedOrderExternalID(externalID string, trades ...*domain.Trade) { + for _, trade := range trades { + cloned := cloneTrade(trade) + r.byOrderExternalID[externalID] = append(r.byOrderExternalID[externalID], cloned) + } +} + +func (r *recordingTradeRepo) Create(_ context.Context, trade *domain.Trade) error { + cloned := cloneTrade(trade) + if cloned.ID == uuid.Nil { + cloned.ID = uuid.New() + } + if cloned.CreatedAt.IsZero() { + cloned.CreatedAt = time.Now().UTC() + } + *trade = *cloned + r.created = append(r.created, cloneTrade(cloned)) + if trade.OrderID != nil && r.orders != nil { + if order, err := r.orders.Get(context.Background(), *trade.OrderID); err == nil { + r.byOrderExternalID[order.ExternalID] = append(r.byOrderExternalID[order.ExternalID], cloneTrade(cloned)) + } + } + return nil +} + +func (r *recordingTradeRepo) List(_ context.Context, _ repository.TradeFilter, _, _ int) ([]domain.Trade, error) { + var trades []domain.Trade + for _, bucket := range r.byOrderExternalID { + for _, trade := range bucket { + trades = append(trades, *cloneTrade(trade)) + } + } + return trades, nil +} + +func (r *recordingTradeRepo) Count(ctx context.Context, filter repository.TradeFilter) (int, error) { + trades, err := r.List(ctx, filter, 0, 0) + if err != nil { + return 0, err + } + return len(trades), nil +} + +func (r *recordingTradeRepo) GetByOrder(_ context.Context, orderID uuid.UUID, _ repository.TradeFilter, _, _ int) ([]domain.Trade, error) { + if r.orders == nil { + return nil, nil + } + order, err := r.orders.Get(context.Background(), orderID) + if err != nil { + return nil, err + } + var trades []domain.Trade + for _, trade := range r.byOrderExternalID[order.ExternalID] { + trades = append(trades, *cloneTrade(trade)) + } + return trades, nil +} + +func (r *recordingTradeRepo) GetByPosition(_ context.Context, _ uuid.UUID, _ repository.TradeFilter, _, _ int) ([]domain.Trade, error) { + return nil, nil +} + +type auditLogRepoStub struct { + entries []*domain.AuditLogEntry +} + +func (r *auditLogRepoStub) Create(_ context.Context, entry *domain.AuditLogEntry) error { + cloned := *entry + if len(entry.Details) > 0 { + cloned.Details = append([]byte(nil), entry.Details...) + } + r.entries = append(r.entries, &cloned) + return nil +} +func (r *auditLogRepoStub) Query(context.Context, repository.AuditLogFilter, int, int) ([]domain.AuditLogEntry, error) { + var out []domain.AuditLogEntry + for _, entry := range r.entries { + out = append(out, *entry) + } + return out, nil +} +func (r *auditLogRepoStub) Count(context.Context, repository.AuditLogFilter) (int, error) { + return len(r.entries), nil +} + +func TestAlpacaReconcilerReconcile_ImportsOrdersPositionsAndFills(t *testing.T) { + t.Parallel() + + strategyID := uuid.New() + strategies := &recordingStrategyRepo{list: []domain.Strategy{{ + ID: strategyID, + Name: "SNAL strategy", + Ticker: "SNAL", + MarketType: domain.MarketTypeStock, + Status: domain.StrategyStatusActive, + IsPaper: false, + }}} + orders := newRecordingOrderRepo() + positions := newRecordingPositionRepo() + trades := newRecordingTradeRepo(orders) + broker := &alpacaReconciliationBrokerStub{ + positions: []domain.Position{{ + Ticker: "SNAL", + Side: domain.PositionSideLong, + Quantity: 200, + AvgEntry: 0.92, + CurrentPrice: float64Ptr(0.7611), + UnrealizedPnL: float64Ptr(-31.78), + }}, + orders: []BrokerOrderSnapshot{{ + ExternalID: "e8405b49-6140-46b5-a78a-7305f1086cd1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 200, + FilledQuantity: 200, + FilledAvgPrice: float64Ptr(0.92), + LimitPrice: float64Ptr(50), + Status: domain.OrderStatusFilled, + SubmittedAt: timePtr(time.Date(2026, 4, 15, 19, 20, 2, 451351000, time.UTC)), + FilledAt: timePtr(time.Date(2026, 4, 15, 19, 20, 4, 943982000, time.UTC)), + Broker: "alpaca", + StrategyIDHint: &strategyID, + }}, + fills: []BrokerFillSnapshot{{ + ActivityID: "20260415152002662::04a8500c-8992-4db5-afca-6cd1b74629be", + ExternalID: "e8405b49-6140-46b5-a78a-7305f1086cd1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 186, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 2, 662680000, time.UTC), + OrderStatus: domain.OrderStatusPartial, + }, { + ActivityID: "20260415152004943::352a24e1-0d2b-469a-9765-43ed6889ca48", + ExternalID: "e8405b49-6140-46b5-a78a-7305f1086cd1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 3, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 4, 943982000, time.UTC), + OrderStatus: domain.OrderStatusFilled, + }}, + } + + audit := &auditLogRepoStub{} + reconciler := NewAlpacaReconciler(AlpacaReconcilerDeps{ + Broker: broker, + StrategyRepo: strategies, + OrderRepo: orders, + PositionRepo: positions, + TradeRepo: trades, + AuditLogRepo: audit, + Logger: slog.New(slog.NewTextHandler(testWriter{t}, nil)), + }) + + summary, err := reconciler.Reconcile(context.Background()) + if err != nil { + t.Fatalf("Reconcile() error = %v", err) + } + if summary.OrdersCreated != 1 { + t.Fatalf("OrdersCreated = %d, want 1", summary.OrdersCreated) + } + if summary.PositionsCreated != 1 { + t.Fatalf("PositionsCreated = %d, want 1", summary.PositionsCreated) + } + if summary.TradesCreated != 2 { + t.Fatalf("TradesCreated = %d, want 2", summary.TradesCreated) + } + if len(orders.created) != 1 { + t.Fatalf("len(orders.created) = %d, want 1", len(orders.created)) + } + if orders.created[0].ExternalID != "e8405b49-6140-46b5-a78a-7305f1086cd1" { + t.Fatalf("created order external id = %q", orders.created[0].ExternalID) + } + if orders.created[0].StrategyID == nil || *orders.created[0].StrategyID != strategyID { + t.Fatalf("created order strategy id = %v, want %s", orders.created[0].StrategyID, strategyID) + } + if len(positions.created) != 1 { + t.Fatalf("len(positions.created) = %d, want 1", len(positions.created)) + } + if positions.created[0].Ticker != "SNAL" { + t.Fatalf("created position ticker = %q, want SNAL", positions.created[0].Ticker) + } + if len(trades.created) != 2 { + t.Fatalf("len(trades.created) = %d, want 2", len(trades.created)) + } + for _, trade := range trades.created { + if trade.PositionID == nil { + t.Fatalf("trade %+v missing position id", trade) + } + if trade.OrderID == nil { + t.Fatalf("trade %+v missing order id", trade) + } + } + if len(audit.entries) != 1 { + t.Fatalf("len(audit.entries) = %d, want 1", len(audit.entries)) + } + if audit.entries[0].EventType != "alpaca_reconcile.completed" { + t.Fatalf("audit event type = %q, want alpaca_reconcile.completed", audit.entries[0].EventType) + } + var details map[string]any + if err := json.Unmarshal(audit.entries[0].Details, &details); err != nil { + t.Fatalf("unmarshal audit details: %v", err) + } + if got := details["orders_created"]; got != float64(1) { + t.Fatalf("audit orders_created = %v, want 1", got) + } + if got := details["positions_created"]; got != float64(1) { + t.Fatalf("audit positions_created = %v, want 1", got) + } + if got := details["trades_created"]; got != float64(2) { + t.Fatalf("audit trades_created = %v, want 2", got) + } +} + +func TestAlpacaReconcilerReconcile_UpdatesExistingRecordsAndSkipsKnownFills(t *testing.T) { + t.Parallel() + + strategyID := uuid.New() + existingOrderID := uuid.New() + existingPositionID := uuid.New() + existingOrder := &domain.Order{ + ID: existingOrderID, + StrategyID: &strategyID, + ExternalID: "existing-order", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 200, + FilledQuantity: 186, + Status: domain.OrderStatusPartial, + Broker: "alpaca", + } + existingPosition := &domain.Position{ + ID: existingPositionID, + StrategyID: &strategyID, + Ticker: "SNAL", + Side: domain.PositionSideLong, + Quantity: 186, + AvgEntry: 0.92, + } + strategies := &recordingStrategyRepo{list: []domain.Strategy{{ + ID: strategyID, + Name: "SNAL strategy", + Ticker: "SNAL", + MarketType: domain.MarketTypeStock, + Status: domain.StrategyStatusActive, + IsPaper: false, + }}} + orders := newRecordingOrderRepo(existingOrder) + positions := newRecordingPositionRepo(existingPosition) + trades := newRecordingTradeRepo(orders) + trades.seedOrderExternalID("existing-order", &domain.Trade{ + ID: uuid.New(), + OrderID: &existingOrderID, + PositionID: &existingPositionID, + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 186, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 2, 662680000, time.UTC), + }) + broker := &alpacaReconciliationBrokerStub{ + positions: []domain.Position{{ + Ticker: "SNAL", + Side: domain.PositionSideLong, + Quantity: 200, + AvgEntry: 0.92, + CurrentPrice: float64Ptr(0.7611), + UnrealizedPnL: float64Ptr(-31.78), + }}, + orders: []BrokerOrderSnapshot{{ + ExternalID: "existing-order", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 200, + FilledQuantity: 200, + FilledAvgPrice: float64Ptr(0.92), + Status: domain.OrderStatusFilled, + Broker: "alpaca", + StrategyIDHint: &strategyID, + SubmittedAt: timePtr(time.Date(2026, 4, 15, 19, 20, 2, 451351000, time.UTC)), + FilledAt: timePtr(time.Date(2026, 4, 15, 19, 20, 4, 943982000, time.UTC)), + }}, + fills: []BrokerFillSnapshot{{ + ActivityID: "known-fill", + ExternalID: "existing-order", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 186, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 2, 662680000, time.UTC), + OrderStatus: domain.OrderStatusPartial, + }, { + ActivityID: "new-fill", + ExternalID: "existing-order", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 14, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 4, 943982000, time.UTC), + OrderStatus: domain.OrderStatusFilled, + }}, + } + + audit := &auditLogRepoStub{} + reconciler := NewAlpacaReconciler(AlpacaReconcilerDeps{ + Broker: broker, + StrategyRepo: strategies, + OrderRepo: orders, + PositionRepo: positions, + TradeRepo: trades, + AuditLogRepo: audit, + Logger: slog.New(slog.NewTextHandler(testWriter{t}, nil)), + }) + + summary, err := reconciler.Reconcile(context.Background()) + if err != nil { + t.Fatalf("Reconcile() error = %v", err) + } + if summary.OrdersUpdated != 1 { + t.Fatalf("OrdersUpdated = %d, want 1", summary.OrdersUpdated) + } + if summary.PositionsUpdated != 1 { + t.Fatalf("PositionsUpdated = %d, want 1", summary.PositionsUpdated) + } + if summary.TradesCreated != 1 { + t.Fatalf("TradesCreated = %d, want 1", summary.TradesCreated) + } + if len(orders.updated) != 1 { + t.Fatalf("len(orders.updated) = %d, want 1", len(orders.updated)) + } + if orders.updated[0].Status != domain.OrderStatusFilled { + t.Fatalf("updated order status = %q, want filled", orders.updated[0].Status) + } + if orders.updated[0].FilledQuantity != 200 { + t.Fatalf("updated order filled quantity = %v, want 200", orders.updated[0].FilledQuantity) + } + if len(positions.updated) != 1 { + t.Fatalf("len(positions.updated) = %d, want 1", len(positions.updated)) + } + if positions.updated[0].Quantity != 200 { + t.Fatalf("updated position quantity = %v, want 200", positions.updated[0].Quantity) + } + if len(trades.created) != 1 { + t.Fatalf("len(trades.created) = %d, want 1", len(trades.created)) + } + if trades.created[0].Quantity != 14 { + t.Fatalf("new trade quantity = %v, want 14", trades.created[0].Quantity) + } + if len(audit.entries) != 1 { + t.Fatalf("len(audit.entries) = %d, want 1", len(audit.entries)) + } + var details map[string]any + if err := json.Unmarshal(audit.entries[0].Details, &details); err != nil { + t.Fatalf("unmarshal audit details: %v", err) + } + if got := details["orders_updated"]; got != float64(1) { + t.Fatalf("audit orders_updated = %v, want 1", got) + } + if got := details["positions_updated"]; got != float64(1) { + t.Fatalf("audit positions_updated = %v, want 1", got) + } + if got := details["trades_created"]; got != float64(1) { + t.Fatalf("audit trades_created = %v, want 1", got) + } +} + +func TestAlpacaReconcilerReconcile_ReturnsErrorWhenBrokerFails(t *testing.T) { + t.Parallel() + + reconciler := NewAlpacaReconciler(AlpacaReconcilerDeps{ + Broker: &alpacaReconciliationBrokerStub{positionsErr: errors.New("boom")}, + OrderRepo: newRecordingOrderRepo(), + PositionRepo: newRecordingPositionRepo(), + TradeRepo: newRecordingTradeRepo(newRecordingOrderRepo()), + Logger: slog.New(slog.NewTextHandler(testWriter{t}, nil)), + }) + + _, err := reconciler.Reconcile(context.Background()) + if err == nil { + t.Fatal("Reconcile() error = nil, want non-nil") + } + if got := err.Error(); got != "alpaca_reconcile: fetch positions: boom" { + t.Fatalf("Reconcile() error = %q, want wrapped fetch positions error", got) + } +} + +func TestAlpacaReconcilerVerify_NormalizesBrokerOrderPrecisionToStorage(t *testing.T) { + t.Parallel() + + strategyID := uuid.New() + orders := newRecordingOrderRepo(&domain.Order{ + ID: uuid.New(), + StrategyID: &strategyID, + ExternalID: "expired-order", + Ticker: "DAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 215.01595699, + FilledQuantity: 0, + Status: domain.OrderStatusCancelled, + Broker: "alpaca", + }) + reconciler := NewAlpacaReconciler(AlpacaReconcilerDeps{ + Broker: &alpacaReconciliationBrokerStub{ + orders: []BrokerOrderSnapshot{{ + ExternalID: "expired-order", + Ticker: "DAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 215.015956989, + FilledQuantity: 0, + Status: domain.OrderStatusCancelled, + Broker: "alpaca", + }}, + }, + OrderRepo: orders, + PositionRepo: newRecordingPositionRepo(), + TradeRepo: newRecordingTradeRepo(orders), + Logger: slog.New(slog.NewTextHandler(testWriter{t}, nil)), + }) + + report, err := reconciler.Verify(context.Background()) + if err != nil { + t.Fatalf("Verify() error = %v", err) + } + if !report.Verified { + t.Fatalf("Verified = false, mismatches = %#v", report.Mismatches) + } + if len(report.Mismatches) != 0 { + t.Fatalf("len(Mismatches) = %d, want 0", len(report.Mismatches)) + } +} + +func TestAlpacaReconcilerVerify_IgnoresVolatilePositionMarkToMarketFields(t *testing.T) { + t.Parallel() + + positions := newRecordingPositionRepo(&domain.Position{ + ID: uuid.New(), + Ticker: "SNAL", + Side: domain.PositionSideLong, + Quantity: 200, + AvgEntry: 0.92, + CurrentPrice: float64Ptr(0.7727), + UnrealizedPnL: float64Ptr(-29.46), + }) + orders := newRecordingOrderRepo() + reconciler := NewAlpacaReconciler(AlpacaReconcilerDeps{ + Broker: &alpacaReconciliationBrokerStub{ + positions: []domain.Position{{ + Ticker: "SNAL", + Side: domain.PositionSideLong, + Quantity: 200, + AvgEntry: 0.92, + CurrentPrice: float64Ptr(0.7695), + UnrealizedPnL: float64Ptr(-30.10), + }}, + }, + OrderRepo: orders, + PositionRepo: positions, + TradeRepo: newRecordingTradeRepo(orders), + Logger: slog.New(slog.NewTextHandler(testWriter{t}, nil)), + }) + + report, err := reconciler.Verify(context.Background()) + if err != nil { + t.Fatalf("Verify() error = %v", err) + } + if !report.Verified { + t.Fatalf("Verified = false, mismatches = %#v", report.Mismatches) + } + if len(report.Mismatches) != 0 { + t.Fatalf("len(Mismatches) = %d, want 0", len(report.Mismatches)) + } +} + +func TestAlpacaReconcilerReconcile_CreatesDistinctTradesForDuplicateExecutionFieldsWhenActivityIDsDiffer(t *testing.T) { + t.Parallel() + + strategyID := uuid.New() + orderID := uuid.New() + orders := newRecordingOrderRepo(&domain.Order{ + ID: orderID, + StrategyID: &strategyID, + ExternalID: "order-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 20, + Status: domain.OrderStatusPartial, + Broker: "alpaca", + }) + trades := newRecordingTradeRepo(orders) + reconciler := NewAlpacaReconciler(AlpacaReconcilerDeps{ + Broker: &alpacaReconciliationBrokerStub{ + orders: []BrokerOrderSnapshot{{ + ExternalID: "order-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 20, + FilledQuantity: 20, + Status: domain.OrderStatusFilled, + Broker: "alpaca", + }}, + fills: []BrokerFillSnapshot{{ + ActivityID: "fill-1", + ExternalID: "order-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 10, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 2, 662680000, time.UTC), + OrderStatus: domain.OrderStatusPartial, + }, { + ActivityID: "fill-2", + ExternalID: "order-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 10, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 2, 662680000, time.UTC), + OrderStatus: domain.OrderStatusFilled, + }}, + }, + OrderRepo: orders, + PositionRepo: newRecordingPositionRepo(), + TradeRepo: trades, + Logger: slog.New(slog.NewTextHandler(testWriter{t}, nil)), + }) + + summary, err := reconciler.Reconcile(context.Background()) + if err != nil { + t.Fatalf("Reconcile() error = %v", err) + } + if summary.TradesCreated != 2 { + t.Fatalf("TradesCreated = %d, want 2", summary.TradesCreated) + } + if len(trades.created) != 2 { + t.Fatalf("len(trades.created) = %d, want 2", len(trades.created)) + } + if trades.created[0].ExternalID == trades.created[1].ExternalID { + t.Fatalf("created trade external ids = %q and %q, want distinct activity ids", trades.created[0].ExternalID, trades.created[1].ExternalID) + } +} + +func TestAlpacaReconcilerVerify_UsesTradeExternalIDToDetectDuplicateExecutionFills(t *testing.T) { + t.Parallel() + + trade := &domain.Trade{ + ID: uuid.New(), + ExternalID: "fill-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 10, + Price: 0.92, + ExecutedAt: time.Date(2026, 4, 15, 19, 20, 2, 662680000, time.UTC), + } + orders := newRecordingOrderRepo(&domain.Order{ + ID: uuid.New(), + ExternalID: "order-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + OrderType: domain.OrderTypeLimit, + Quantity: 20, + Status: domain.OrderStatusFilled, + Broker: "alpaca", + }) + trades := newRecordingTradeRepo(orders) + trades.seedOrderExternalID("order-1", trade) + + reconciler := NewAlpacaReconciler(AlpacaReconcilerDeps{ + Broker: &alpacaReconciliationBrokerStub{ + fills: []BrokerFillSnapshot{{ + ActivityID: "fill-1", + ExternalID: "order-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 10, + Price: 0.92, + ExecutedAt: trade.ExecutedAt, + OrderStatus: domain.OrderStatusPartial, + }, { + ActivityID: "fill-2", + ExternalID: "order-1", + Ticker: "SNAL", + Side: domain.OrderSideBuy, + Quantity: 10, + Price: 0.92, + ExecutedAt: trade.ExecutedAt, + OrderStatus: domain.OrderStatusFilled, + }}, + }, + OrderRepo: orders, + PositionRepo: newRecordingPositionRepo(), + TradeRepo: trades, + Logger: slog.New(slog.NewTextHandler(testWriter{t}, nil)), + }) + + report, err := reconciler.Verify(context.Background()) + if err != nil { + t.Fatalf("Verify() error = %v", err) + } + if report.Verified { + t.Fatalf("Verified = true, want false with missing duplicate fill trade") + } + if report.MissingTrades != 1 { + t.Fatalf("MissingTrades = %d, want 1", report.MissingTrades) + } +} + +func TestJobOrchestratorRegisterAll_IncludesAlpacaReconcileJob(t *testing.T) { + t.Parallel() + + orch := NewJobOrchestrator(OrchestratorDeps{}) + orch.RegisterAll() + + status := singleJobStatus(t, orch, "alpaca_reconcile") + if status.Name != "alpaca_reconcile" { + t.Fatalf("status.Name = %q, want alpaca_reconcile", status.Name) + } +} + +type testWriter struct{ t *testing.T } + +func (w testWriter) Write(p []byte) (int, error) { + w.t.Helper() + w.t.Log(strings.TrimSpace(string(p))) + return len(p), nil +} + +func cloneOrder(order *domain.Order) *domain.Order { + if order == nil { + return nil + } + cloned := *order + if order.StrategyID != nil { + id := *order.StrategyID + cloned.StrategyID = &id + } + if order.PipelineRunID != nil { + id := *order.PipelineRunID + cloned.PipelineRunID = &id + } + if order.LimitPrice != nil { + value := *order.LimitPrice + cloned.LimitPrice = &value + } + if order.StopPrice != nil { + value := *order.StopPrice + cloned.StopPrice = &value + } + if order.FilledAvgPrice != nil { + value := *order.FilledAvgPrice + cloned.FilledAvgPrice = &value + } + if order.SubmittedAt != nil { + value := *order.SubmittedAt + cloned.SubmittedAt = &value + } + if order.FilledAt != nil { + value := *order.FilledAt + cloned.FilledAt = &value + } + return &cloned +} + +func clonePosition(position *domain.Position) *domain.Position { + if position == nil { + return nil + } + cloned := *position + if position.StrategyID != nil { + id := *position.StrategyID + cloned.StrategyID = &id + } + if position.CurrentPrice != nil { + value := *position.CurrentPrice + cloned.CurrentPrice = &value + } + if position.UnrealizedPnL != nil { + value := *position.UnrealizedPnL + cloned.UnrealizedPnL = &value + } + if position.StopLoss != nil { + value := *position.StopLoss + cloned.StopLoss = &value + } + if position.TakeProfit != nil { + value := *position.TakeProfit + cloned.TakeProfit = &value + } + if position.ClosedAt != nil { + value := *position.ClosedAt + cloned.ClosedAt = &value + } + if position.Expiry != nil { + value := *position.Expiry + cloned.Expiry = &value + } + return &cloned +} + +func cloneTrade(trade *domain.Trade) *domain.Trade { + if trade == nil { + return nil + } + cloned := *trade + cloned.ExternalID = trade.ExternalID + if trade.OrderID != nil { + id := *trade.OrderID + cloned.OrderID = &id + } + if trade.PositionID != nil { + id := *trade.PositionID + cloned.PositionID = &id + } + return &cloned +} + +func paginateOrders(items []domain.Order, limit, offset int) []domain.Order { + if offset >= len(items) { + return nil + } + if limit <= 0 { + limit = len(items) + } + end := offset + limit + if end > len(items) { + end = len(items) + } + return items[offset:end] +} + +func paginatePositions(items []domain.Position, limit, offset int) []domain.Position { + if offset >= len(items) { + return nil + } + if limit <= 0 { + limit = len(items) + } + end := offset + limit + if end > len(items) { + end = len(items) + } + return items[offset:end] +} + +func float64Ptr(v float64) *float64 { return &v } +func timePtr(v time.Time) *time.Time { return &v } diff --git a/internal/automation/jobs_broker_reconciliation.go b/internal/automation/jobs_broker_reconciliation.go new file mode 100644 index 00000000..28b26e0d --- /dev/null +++ b/internal/automation/jobs_broker_reconciliation.go @@ -0,0 +1,55 @@ +package automation + +import ( + "context" + "fmt" + "log/slog" + + "github.com/PatrickFanella/get-rich-quick/internal/scheduler" +) + +var alpacaReconcileSpec = scheduler.ScheduleSpec{ + Type: scheduler.ScheduleTypeCron, + Cron: "*/5 * * * *", +} + +func (o *JobOrchestrator) registerBrokerReconciliationJobs() { + o.Register( + "alpaca_reconcile", + "Reconcile Alpaca broker positions, orders, and fills into local state", + alpacaReconcileSpec, + o.alpacaReconcile, + ) +} + +func (o *JobOrchestrator) alpacaReconcile(ctx context.Context) error { + if o.deps.AlpacaReconciler == nil { + o.logger.Info("alpaca_reconcile: skipped — reconciler not configured") + if o.metrics != nil { + o.metrics.RecordAlpacaReconcileRun("skipped") + } + return nil + } + + o.logger.Info("alpaca_reconcile: starting") + summary, err := o.deps.AlpacaReconciler.Reconcile(ctx) + if err != nil { + if o.metrics != nil { + o.metrics.RecordAlpacaReconcileRun("error") + } + return fmt.Errorf("alpaca_reconcile: %w", err) + } + o.SetLastSummary("alpaca_reconcile", summary.Map()) + if o.metrics != nil { + o.metrics.RecordAlpacaReconcileRun("success") + } + + o.logger.Info("alpaca_reconcile: complete", + slog.Int("orders_created", summary.OrdersCreated), + slog.Int("orders_updated", summary.OrdersUpdated), + slog.Int("positions_created", summary.PositionsCreated), + slog.Int("positions_updated", summary.PositionsUpdated), + slog.Int("trades_created", summary.TradesCreated), + ) + return nil +} diff --git a/internal/automation/orchestrator.go b/internal/automation/orchestrator.go index afd054da..d9258e7d 100644 --- a/internal/automation/orchestrator.go +++ b/internal/automation/orchestrator.go @@ -49,6 +49,7 @@ type OrchestratorDeps struct { Universe *universe.Universe Polygon *polygon.Client DataService *data.DataService + AlpacaReconciler *AlpacaReconciler OptionsProvider data.OptionsDataProvider LLMProvider llm.Provider EmbeddingProvider embedding.Provider // optional; nil = skip embedding during triage @@ -78,6 +79,7 @@ type RegisteredJob struct { StartedAt *time.Time LastRun *time.Time LastResult string + LastSummary map[string]int LastError string LastErrorAt *time.Time RunCount int @@ -94,6 +96,7 @@ type JobStatus struct { Schedule string `json:"schedule"` LastRun *time.Time `json:"last_run,omitempty"` LastResult string `json:"last_result"` + LastSummary map[string]int `json:"last_summary,omitempty"` LastError string `json:"last_error,omitempty"` LastErrorAt *time.Time `json:"last_error_at,omitempty"` RunCount int `json:"run_count"` @@ -108,6 +111,7 @@ type JobStatus struct { // It is defined here as an interface to avoid an import cycle. type AutomationJobMetrics interface { RecordAutomationJobError(jobName string) + RecordAlpacaReconcileRun(result string) } // ReportWorkerMetrics captures report worker success/error emission. @@ -163,6 +167,14 @@ func (o *JobOrchestrator) SetConsecutiveFailures(name string, n int) { } } +func (o *JobOrchestrator) SetLastSummary(name string, summary map[string]int) { + if job, ok := o.jobs[name]; ok { + job.mu.Lock() + job.LastSummary = cloneSummary(summary) + job.mu.Unlock() + } +} + // Register adds a job to the registry. func (o *JobOrchestrator) Register(name, description string, spec scheduler.ScheduleSpec, fn func(ctx context.Context) error, dependsOn ...string) { o.jobs[name] = &RegisteredJob{ @@ -177,6 +189,7 @@ func (o *JobOrchestrator) Register(name, description string, spec scheduler.Sche // RegisterAll registers all automated jobs from every job group. func (o *JobOrchestrator) RegisterAll() { + o.registerBrokerReconciliationJobs() o.registerMarketJobs() o.registerPreMarketJobs() o.registerPostMarketJobs() @@ -235,6 +248,7 @@ func (o *JobOrchestrator) Status() []JobStatus { Schedule: job.Schedule.Describe(), LastRun: job.LastRun, LastResult: job.LastResult, + LastSummary: cloneSummary(job.LastSummary), LastError: job.LastError, LastErrorAt: job.LastErrorAt, RunCount: job.RunCount, @@ -530,3 +544,14 @@ func (o *JobOrchestrator) hydrateFromDB() { o.logger.Info("automation: hydrated job stats from DB", slog.Int("jobs", len(summaries))) } + +func cloneSummary(summary map[string]int) map[string]int { + if len(summary) == 0 { + return nil + } + cloned := make(map[string]int, len(summary)) + for key, value := range summary { + cloned[key] = value + } + return cloned +} diff --git a/internal/automation/orchestrator_test.go b/internal/automation/orchestrator_test.go index 7664522b..15bdb41a 100644 --- a/internal/automation/orchestrator_test.go +++ b/internal/automation/orchestrator_test.go @@ -3,6 +3,7 @@ package automation import ( "context" "errors" + "log/slog" "testing" "time" @@ -144,6 +145,66 @@ func TestJobOrchestratorWrapAndRun_AutoDisabledJobsAreSkipped(t *testing.T) { } } +type stubAutomationMetrics struct { + alpacaRuns map[string]int +} + +func (m *stubAutomationMetrics) RecordAutomationJobError(string) {} + +func (m *stubAutomationMetrics) RecordAlpacaReconcileRun(result string) { + if m.alpacaRuns == nil { + m.alpacaRuns = make(map[string]int) + } + m.alpacaRuns[result]++ +} + +func TestJobOrchestratorStatus_IncludesLastSummary(t *testing.T) { + t.Parallel() + + orch := NewJobOrchestrator(OrchestratorDeps{}) + orch.Register("alpaca_reconcile", "test job", schedulerSpecEveryMinute(), func(context.Context) error { return nil }) + orch.SetLastSummary("alpaca_reconcile", map[string]int{"orders_created": 2, "trades_created": 3}) + + status := singleJobStatus(t, orch, "alpaca_reconcile") + if status.LastSummary == nil { + t.Fatal("LastSummary = nil, want populated") + } + if status.LastSummary["orders_created"] != 2 { + t.Fatalf("orders_created = %d, want 2", status.LastSummary["orders_created"]) + } + status.LastSummary["orders_created"] = 99 + statusAgain := singleJobStatus(t, orch, "alpaca_reconcile") + if statusAgain.LastSummary["orders_created"] != 2 { + t.Fatalf("mutated summary leaked into orchestrator: %d", statusAgain.LastSummary["orders_created"]) + } +} + +func TestJobOrchestratorAlpacaReconcileRecordsMetricsAndSummary(t *testing.T) { + t.Parallel() + + metrics := &stubAutomationMetrics{} + orch := NewJobOrchestrator(OrchestratorDeps{Logger: slog.Default()}) + orch.WithJobMetrics(metrics) + orch.Register("alpaca_reconcile", "test job", schedulerSpecEveryMinute(), func(context.Context) error { + orch.SetLastSummary("alpaca_reconcile", map[string]int{"orders_created": 1}) + metrics.RecordAlpacaReconcileRun("success") + return nil + }) + + if err := orch.RunJob(context.Background(), "alpaca_reconcile"); err != nil { + t.Fatalf("RunJob() error = %v", err) + } + waitForJobRuns(t, orch, "alpaca_reconcile", 1) + + status := singleJobStatus(t, orch, "alpaca_reconcile") + if status.LastSummary == nil || status.LastSummary["orders_created"] != 1 { + t.Fatalf("LastSummary = %#v, want orders_created=1", status.LastSummary) + } + if metrics.alpacaRuns["success"] != 1 { + t.Fatalf("alpaca success runs = %d, want 1", metrics.alpacaRuns["success"]) + } +} + func waitForJobRuns(t *testing.T, orch *JobOrchestrator, jobName string, want int) { t.Helper() deadline := time.Now().Add(2 * time.Second) diff --git a/internal/cli/automation_test.go b/internal/cli/automation_test.go new file mode 100644 index 00000000..ca6df2b2 --- /dev/null +++ b/internal/cli/automation_test.go @@ -0,0 +1,49 @@ +package cli + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestAutomationAlpacaReconcileCommandRunsAdminEndpoint(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Fatalf("method = %s, want POST", r.Method) + } + if r.URL.Path != "/api/v1/automation/alpaca/reconcile" { + t.Fatalf("path = %s, want /api/v1/automation/alpaca/reconcile", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "summary": map[string]int{ + "orders_created": 2, + "trades_created": 3, + }, + "verification": map[string]int{ + "orders_checked": 2, + }, + }) + })) + defer server.Close() + + stdout, _, err := executeCLI(t, nil, "--api-url", server.URL, "--format", "json", "automation", "alpaca-reconcile") + if err != nil { + t.Fatalf("Execute() error = %v", err) + } + + var out map[string]any + if err := json.Unmarshal([]byte(stdout), &out); err != nil { + t.Fatalf("stdout is not valid JSON: %v\n%s", err, stdout) + } + summary, ok := out["summary"].(map[string]any) + if !ok { + t.Fatalf("summary = %#v, want object", out["summary"]) + } + if summary["orders_created"] != float64(2) { + t.Fatalf("orders_created = %v, want 2", summary["orders_created"]) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index bf135e9d..d1509cb5 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -267,6 +267,7 @@ func NewRootCommand(ctx context.Context, deps Dependencies) *cobra.Command { rootCmd.AddCommand(state.newDashboardCommand()) rootCmd.AddCommand(state.newPortfolioCommand()) rootCmd.AddCommand(state.newRiskCommand()) + rootCmd.AddCommand(state.newAutomationCommand()) rootCmd.AddCommand(state.newMemoriesCommand()) return rootCmd @@ -564,6 +565,46 @@ func (s *rootState) newRiskCommand() *cobra.Command { return commands } +func (s *rootState) newAutomationCommand() *cobra.Command { + commands := &cobra.Command{ + Use: "automation", + Short: "Inspect and trigger automation admin flows", + Long: "Work with automation status and Alpaca reconciliation endpoints through the local API.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + + commands.AddCommand(&cobra.Command{ + Use: "alpaca-reconcile", + Short: "Run Alpaca reconciliation immediately", + Long: "Trigger an immediate Alpaca positions/orders/trades reconciliation and return verification details.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + client, err := s.client() + if err != nil { + return err + } + + var response map[string]any + if err := client.post(cmd.Context(), "/api/v1/automation/alpaca/reconcile", nil, nil, &response); err != nil { + return err + } + + if s.format == formatJSON { + return writeJSON(cmd.OutOrStdout(), response) + } + return writeTable(cmd.OutOrStdout(), []string{"FIELD", "VALUE"}, [][]string{ + {"Summary", fmt.Sprintf("%v", response["summary"])}, + {"Verification", fmt.Sprintf("%v", response["verification"])}, + }) + }, + }) + + return commands +} + func (s *rootState) newMemoriesCommand() *cobra.Command { commands := &cobra.Command{ Use: "memories", diff --git a/internal/data/polygon/client.go b/internal/data/polygon/client.go index 531708b4..7da4ba98 100644 --- a/internal/data/polygon/client.go +++ b/internal/data/polygon/client.go @@ -21,11 +21,13 @@ const ( // Client is a small HTTP client for Polygon.io APIs. type Client struct { - apiKey string - baseURL string - httpClient *http.Client - api *data.APIClient - logger *slog.Logger + apiKey string + baseURL string + httpClient *http.Client + api *data.APIClient + logger *slog.Logger + tickerPageDelay time.Duration + sleeper func(context.Context, time.Duration) error } // ErrorResponse captures Polygon's standard error response shape. @@ -64,11 +66,13 @@ func NewClient(apiKey string, logger *slog.Logger) *Client { api.SetHTTPClient(httpClient) return &Client{ - apiKey: trimmedKey, - baseURL: defaultBaseURL, - httpClient: httpClient, - api: api, - logger: logger, + apiKey: trimmedKey, + baseURL: defaultBaseURL, + httpClient: httpClient, + api: api, + logger: logger, + tickerPageDelay: 12 * time.Second, + sleeper: sleepWithContext, } } @@ -88,6 +92,37 @@ func (c *Client) SetTimeout(timeout time.Duration) { c.httpClient.Timeout = timeout } +// SetTickerPageDelay overrides the inter-page delay used by ListActiveTickers. +func (c *Client) SetTickerPageDelay(delay time.Duration) { + if c == nil || delay <= 0 { + return + } + c.tickerPageDelay = delay +} + +// SetSleeper overrides the delay implementation used by ListActiveTickers. +func (c *Client) SetSleeper(sleeper func(context.Context, time.Duration) error) { + if c == nil || sleeper == nil { + return + } + c.sleeper = sleeper +} + +func sleepWithContext(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + return nil + } + timer := time.NewTimer(delay) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + // Get issues a GET request to the supplied Polygon API path and returns the raw response body. func (c *Client) Get(ctx context.Context, requestPath string, params url.Values) ([]byte, error) { if c == nil { diff --git a/internal/data/polygon/universe.go b/internal/data/polygon/universe.go index 740be979..586ff952 100644 --- a/internal/data/polygon/universe.go +++ b/internal/data/polygon/universe.go @@ -122,10 +122,16 @@ func (c *Client) ListActiveTickers(ctx context.Context, market, tickerType strin // Rate limit pause: Polygon free tier allows 5 req/min, so paginated // reference-ticker requests need roughly 12s spacing to avoid 429s. - select { - case <-ctx.Done(): - return tickers, ctx.Err() - case <-time.After(12 * time.Second): + delay := c.tickerPageDelay + if delay <= 0 { + delay = 12 * time.Second + } + sleeper := c.sleeper + if sleeper == nil { + sleeper = sleepWithContext + } + if err := sleeper(ctx, delay); err != nil { + return tickers, err } } diff --git a/internal/data/polygon/universe_test.go b/internal/data/polygon/universe_test.go index 9a821cf2..eb03ed7b 100644 --- a/internal/data/polygon/universe_test.go +++ b/internal/data/polygon/universe_test.go @@ -5,29 +5,28 @@ import ( "fmt" "net/http" "net/http/httptest" + "reflect" "testing" "time" ) func TestListActiveTickersRespectsFreeTierRateLimit(t *testing.T) { - var firstRequestAt time.Time + t.Parallel() + + var requestedCursors []string + var sleepCalls []time.Duration var serverURL string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") + requestedCursors = append(requestedCursors, r.URL.Query().Get("cursor")) switch r.URL.Query().Get("cursor") { case "": - firstRequestAt = time.Now() _, _ = fmt.Fprintf(w, `{"results":[{"ticker":"AAA","name":"Alpha","primary_exchange":"XNAS","type":"CS","active":true}],"next_url":"%s/v3/reference/tickers?cursor=page-2"}`, serverURL, ) case "page-2": - if time.Since(firstRequestAt) < 11*time.Second { - w.WriteHeader(http.StatusTooManyRequests) - _, _ = w.Write([]byte(`{"status":"ERROR","request_id":"req-rate","error":"rate limit exceeded"}`)) - return - } _, _ = w.Write([]byte(`{"results":[{"ticker":"BBB","name":"Beta","primary_exchange":"XNYS","type":"CS","active":true}]}`)) default: w.WriteHeader(http.StatusBadRequest) @@ -38,6 +37,11 @@ func TestListActiveTickersRespectsFreeTierRateLimit(t *testing.T) { client := NewClient("test-key", discardLogger()) client.baseURL = server.URL + client.SetTickerPageDelay(12 * time.Second) + client.SetSleeper(func(_ context.Context, d time.Duration) error { + sleepCalls = append(sleepCalls, d) + return nil + }) tickers, err := client.ListActiveTickers(context.Background(), "stocks", "CS") if err != nil { @@ -49,4 +53,10 @@ func TestListActiveTickersRespectsFreeTierRateLimit(t *testing.T) { if tickers[0].Ticker != "AAA" || tickers[1].Ticker != "BBB" { t.Fatalf("ListActiveTickers() tickers = %#v, want AAA then BBB", tickers) } + if !reflect.DeepEqual(requestedCursors, []string{"", "page-2"}) { + t.Fatalf("requested cursors = %#v, want first page then page-2", requestedCursors) + } + if !reflect.DeepEqual(sleepCalls, []time.Duration{12 * time.Second}) { + t.Fatalf("sleep calls = %#v, want [12s]", sleepCalls) + } } diff --git a/internal/discovery/deploy.go b/internal/discovery/deploy.go new file mode 100644 index 00000000..b8f6d0eb --- /dev/null +++ b/internal/discovery/deploy.go @@ -0,0 +1,86 @@ +package discovery + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgconn" + + "github.com/PatrickFanella/get-rich-quick/internal/domain" + "github.com/PatrickFanella/get-rich-quick/internal/repository" +) + +// CreateOrReusePaperStrategy creates a paper strategy if it does not already +// exist, and returns the existing row when a matching strategy is present. +// +// Matching key: (ticker, market_type, is_paper=true, exact name). +func CreateOrReusePaperStrategy(ctx context.Context, repo repository.StrategyRepository, strategy domain.Strategy) (domain.Strategy, bool, error) { + if !strategy.IsPaper { + if err := repo.Create(ctx, &strategy); err != nil { + return domain.Strategy{}, false, err + } + return strategy, true, nil + } + + existing, err := findExistingPaperStrategy(ctx, repo, strategy) + if err != nil { + return domain.Strategy{}, false, err + } + if existing != nil { + return *existing, false, nil + } + + if err := repo.Create(ctx, &strategy); err != nil { + // Handle races where another runner inserted the same strategy between + // the List and Create calls. + if !isUniqueViolation(err) { + return domain.Strategy{}, false, err + } + + existingAfterConflict, lookupErr := findExistingPaperStrategy(ctx, repo, strategy) + if lookupErr != nil { + return domain.Strategy{}, false, fmt.Errorf("lookup existing strategy after unique conflict: %w", lookupErr) + } + if existingAfterConflict == nil { + return domain.Strategy{}, false, err + } + return *existingAfterConflict, false, nil + } + + return strategy, true, nil +} + +func findExistingPaperStrategy(ctx context.Context, repo repository.StrategyRepository, strategy domain.Strategy) (*domain.Strategy, error) { + isPaper := true + existing, err := repo.List(ctx, repository.StrategyFilter{ + Ticker: strategy.Ticker, + MarketType: strategy.MarketType, + IsPaper: &isPaper, + }, 200, 0) + if err != nil { + return nil, fmt.Errorf("list existing strategies for %s: %w", strategy.Ticker, err) + } + + for i := range existing { + if existing[i].Name == strategy.Name { + copy := existing[i] + return ©, nil + } + } + + return nil, nil +} + +func isUniqueViolation(err error) bool { + if err == nil { + return false + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code == "23505" + } + errText := strings.ToLower(err.Error()) + return strings.Contains(errText, "duplicate key") || strings.Contains(errText, "unique") +} diff --git a/internal/discovery/deploy_test.go b/internal/discovery/deploy_test.go new file mode 100644 index 00000000..a9056adb --- /dev/null +++ b/internal/discovery/deploy_test.go @@ -0,0 +1,224 @@ +package discovery + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/PatrickFanella/get-rich-quick/internal/domain" + "github.com/PatrickFanella/get-rich-quick/internal/repository" +) + +func TestCreateOrReusePaperStrategyCreatesThenReuses(t *testing.T) { + t.Parallel() + + repo := newInMemoryStrategyRepo() + ctx := context.Background() + + strategy := domain.Strategy{ + Name: "discovery: PBM RSI Momentum Breakout", + Ticker: "PBM", + MarketType: domain.MarketTypeStock, + IsPaper: true, + Status: domain.StrategyStatusActive, + Config: json.RawMessage(`{"rules_engine":{"name":"pbm"}}`), + } + + created, didCreate, err := CreateOrReusePaperStrategy(ctx, repo, strategy) + if err != nil { + t.Fatalf("CreateOrReusePaperStrategy(first) error = %v", err) + } + if !didCreate { + t.Fatal("first call should create strategy") + } + if created.ID == uuid.Nil { + t.Fatal("created strategy id should be set") + } + + reused, didCreate, err := CreateOrReusePaperStrategy(ctx, repo, strategy) + if err != nil { + t.Fatalf("CreateOrReusePaperStrategy(second) error = %v", err) + } + if didCreate { + t.Fatal("second call should reuse existing strategy") + } + if reused.ID != created.ID { + t.Fatalf("reused strategy id = %s, want %s", reused.ID, created.ID) + } + + got := repo.countByKey(strategy.Ticker, strategy.MarketType, strategy.Name, true) + if got != 1 { + t.Fatalf("strategy count for key = %d, want 1", got) + } +} + +func TestCreateOrReusePaperStrategyHandlesUniqueConflictByRequery(t *testing.T) { + t.Parallel() + + repo := newInMemoryStrategyRepo() + repo.injectConflictOnce = true + ctx := context.Background() + + strategy := domain.Strategy{ + Name: "options: QQQ bull_put_spread", + Ticker: "QQQ", + MarketType: domain.MarketTypeOptions, + IsPaper: true, + Status: domain.StrategyStatusActive, + Config: json.RawMessage(`{"options_rules":{"strategy_type":"bull_put_spread"}}`), + } + + reused, didCreate, err := CreateOrReusePaperStrategy(ctx, repo, strategy) + if err != nil { + t.Fatalf("CreateOrReusePaperStrategy() error = %v", err) + } + if didCreate { + t.Fatal("conflict path should return reused existing strategy") + } + if reused.ID == uuid.Nil { + t.Fatal("reused strategy id should be set") + } + + got := repo.countByKey(strategy.Ticker, strategy.MarketType, strategy.Name, true) + if got != 1 { + t.Fatalf("strategy count for key = %d, want 1", got) + } +} + +type inMemoryStrategyRepo struct { + strategies []domain.Strategy + injectConflictOnce bool + conflictTriggered bool +} + +func newInMemoryStrategyRepo() *inMemoryStrategyRepo { + return &inMemoryStrategyRepo{strategies: make([]domain.Strategy, 0)} +} + +func (r *inMemoryStrategyRepo) Create(_ context.Context, strategy *domain.Strategy) error { + if strategy.ID == uuid.Nil { + strategy.ID = uuid.New() + } + + if r.injectConflictOnce && !r.conflictTriggered { + r.conflictTriggered = true + + existing := *strategy + existing.ID = uuid.New() + r.strategies = append(r.strategies, existing) + return errors.New("ERROR: duplicate key value violates unique constraint \"idx_strategies_discovery_unique\" (SQLSTATE 23505)") + } + + r.strategies = append(r.strategies, *strategy) + return nil +} + +func (r *inMemoryStrategyRepo) Get(_ context.Context, id uuid.UUID) (*domain.Strategy, error) { + for i := range r.strategies { + if r.strategies[i].ID == id { + copy := r.strategies[i] + return ©, nil + } + } + return nil, repository.ErrNotFound +} + +func (r *inMemoryStrategyRepo) List(_ context.Context, filter repository.StrategyFilter, limit, offset int) ([]domain.Strategy, error) { + var filtered []domain.Strategy + for _, strategy := range r.strategies { + if filter.Ticker != "" && strategy.Ticker != filter.Ticker { + continue + } + if filter.MarketType != "" && strategy.MarketType != filter.MarketType { + continue + } + if filter.Status != "" && strategy.Status != filter.Status { + continue + } + if filter.IsPaper != nil && strategy.IsPaper != *filter.IsPaper { + continue + } + filtered = append(filtered, strategy) + } + + sort.Slice(filtered, func(i, j int) bool { + return filtered[i].Name < filtered[j].Name + }) + + if offset > len(filtered) { + return []domain.Strategy{}, nil + } + filtered = filtered[offset:] + if limit > 0 && limit < len(filtered) { + filtered = filtered[:limit] + } + return filtered, nil +} + +func (r *inMemoryStrategyRepo) Count(ctx context.Context, filter repository.StrategyFilter) (int, error) { + listed, err := r.List(ctx, filter, 0, 0) + if err != nil { + return 0, err + } + return len(listed), nil +} + +func (r *inMemoryStrategyRepo) Update(_ context.Context, strategy *domain.Strategy) error { + for i := range r.strategies { + if r.strategies[i].ID == strategy.ID { + r.strategies[i] = *strategy + return nil + } + } + return repository.ErrNotFound +} + +func (r *inMemoryStrategyRepo) Delete(_ context.Context, id uuid.UUID) error { + for i := range r.strategies { + if r.strategies[i].ID == id { + r.strategies = append(r.strategies[:i], r.strategies[i+1:]...) + return nil + } + } + return repository.ErrNotFound +} + +func (r *inMemoryStrategyRepo) UpdateThesis(_ context.Context, _ uuid.UUID, _ json.RawMessage) error { + return nil +} + +func (r *inMemoryStrategyRepo) GetThesisRaw(_ context.Context, _ uuid.UUID) (json.RawMessage, error) { + return nil, nil +} + +func (r *inMemoryStrategyRepo) countByKey(ticker string, marketType domain.MarketType, name string, isPaper bool) int { + count := 0 + for _, strategy := range r.strategies { + if strategy.Ticker == ticker && strategy.MarketType == marketType && strategy.Name == name && strategy.IsPaper == isPaper { + count++ + } + } + return count +} + +var _ repository.StrategyRepository = (*inMemoryStrategyRepo)(nil) + +func TestIsUniqueViolationHandlesCommonErrors(t *testing.T) { + t.Parallel() + + if !isUniqueViolation(errors.New("ERROR: duplicate key value violates unique constraint \"foo\"")) { + t.Fatal("expected duplicate key text to be treated as unique violation") + } + if isUniqueViolation(fmt.Errorf("some other error: %w", errors.New("network"))) { + t.Fatal("unexpected unique violation on unrelated error") + } + if !isUniqueViolation(errors.New(strings.ToUpper("unique constraint violation"))) { + t.Fatal("expected case-insensitive unique detection") + } +} diff --git a/internal/discovery/options/generator.go b/internal/discovery/options/generator.go index eab542cc..b9808bc9 100644 --- a/internal/discovery/options/generator.go +++ b/internal/discovery/options/generator.go @@ -3,6 +3,7 @@ package options import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "strings" @@ -22,13 +23,13 @@ The JSON schema is: "entry": { "operator": "AND" | "OR", "conditions": [ - {"field": "", "operator": "", "value": } + {"field": "", "op": "", "value": } ] }, "exit": { "operator": "AND" | "OR", "conditions": [ - {"field": "", "operator": "", "value": } + {"field": "", "op": "", "value": } ] }, "leg_selection": { @@ -117,6 +118,9 @@ func GenerateOptionsStrategy(ctx context.Context, cfg discovery.GeneratorConfig, ) parsed, parseErr := rules.ParseOptions(json.RawMessage(resp.Content)) + if parsed == nil && parseErr == nil { + parseErr = errors.New("rules: empty JSON response") + } if parseErr == nil && parsed != nil { logger.Info("options/generator: strategy generated", slog.String("ticker", candidate.Ticker), @@ -132,12 +136,13 @@ func GenerateOptionsStrategy(ctx context.Context, cfg discovery.GeneratorConfig, slog.Int("attempt", attempt+1), slog.Any("error", parseErr), ) + parseErrText := parseErr.Error() messages = append(messages, llm.Message{Role: "assistant", Content: resp.Content}, llm.Message{Role: "user", Content: fmt.Sprintf( "The JSON you produced failed validation with this error:\n%s\n\nPlease fix the issue and return corrected JSON only.", - parseErr.Error(), + parseErrText, )}, ) } diff --git a/internal/discovery/options/generator_test.go b/internal/discovery/options/generator_test.go new file mode 100644 index 00000000..86ca6531 --- /dev/null +++ b/internal/discovery/options/generator_test.go @@ -0,0 +1,98 @@ +package options + +import ( + "context" + "strings" + "testing" + + "github.com/PatrickFanella/get-rich-quick/internal/discovery" + "github.com/PatrickFanella/get-rich-quick/internal/llm" +) + +func TestGenerateOptionsStrategy_RetriesAfterEmptyResponse(t *testing.T) { + t.Parallel() + + provider := &stubOptionsCompletionProvider{responses: []*llm.CompletionResponse{ + {Content: ""}, + {Content: validOptionsStrategyJSON}, + }} + + got, err := GenerateOptionsStrategy(context.Background(), discovery.GeneratorConfig{ + Provider: provider, + MaxRetries: 1, + }, OptionsScoredCandidate{OptionsScreenResult: OptionsScreenResult{Ticker: "NVDA"}}, nil) + if err != nil { + t.Fatalf("GenerateOptionsStrategy() error = %v, want nil", err) + } + if got == nil { + t.Fatal("GenerateOptionsStrategy() = nil, want config") + } + if string(got.StrategyType) != "bull_put_spread" { + t.Fatalf("GenerateOptionsStrategy().StrategyType = %q, want %q", got.StrategyType, "bull_put_spread") + } + if provider.calls != 2 { + t.Fatalf("provider calls = %d, want 2", provider.calls) + } + if len(provider.requests) != 2 { + t.Fatalf("requests = %d, want 2", len(provider.requests)) + } + msgs := provider.requests[1].Messages + if len(msgs) < 4 { + t.Fatalf("retry messages = %d, want at least 4", len(msgs)) + } + if !strings.Contains(msgs[len(msgs)-1].Content, "rules: empty JSON response") { + t.Fatalf("retry prompt missing empty-response error: %q", msgs[len(msgs)-1].Content) + } +} + +func TestGenerateOptionsStrategy_ReturnsErrorAfterRepeatedEmptyResponses(t *testing.T) { + t.Parallel() + + provider := &stubOptionsCompletionProvider{responses: []*llm.CompletionResponse{ + {Content: ""}, + {Content: ""}, + }} + + got, err := GenerateOptionsStrategy(context.Background(), discovery.GeneratorConfig{ + Provider: provider, + MaxRetries: 1, + }, OptionsScoredCandidate{OptionsScreenResult: OptionsScreenResult{Ticker: "GOOG"}}, nil) + if err == nil { + t.Fatal("GenerateOptionsStrategy() error = nil, want non-nil") + } + if got != nil { + t.Fatalf("GenerateOptionsStrategy() = %#v, want nil", got) + } + if !strings.Contains(err.Error(), "rules: empty JSON response") { + t.Fatalf("GenerateOptionsStrategy() error = %q, want empty-response error", err.Error()) + } + if provider.calls != 2 { + t.Fatalf("provider calls = %d, want 2", provider.calls) + } +} + +func TestOptionsGeneratorSystemPrompt_UsesOpConditionField(t *testing.T) { + t.Parallel() + + if count := strings.Count(optionsGeneratorSystemPrompt, `{"field": "", "op": "", "value": }`); count != 2 { + t.Fatalf("condition field schema appears %d times, want 2", count) + } +} + +type stubOptionsCompletionProvider struct { + responses []*llm.CompletionResponse + requests []llm.CompletionRequest + calls int +} + +func (s *stubOptionsCompletionProvider) Complete(_ context.Context, request llm.CompletionRequest) (*llm.CompletionResponse, error) { + s.requests = append(s.requests, request) + idx := s.calls + s.calls++ + if idx >= len(s.responses) { + return s.responses[len(s.responses)-1], nil + } + return s.responses[idx], nil +} + +const validOptionsStrategyJSON = `{"version":1,"strategy_type":"bull_put_spread","underlying":"NVDA","entry":{"operator":"AND","conditions":[{"field":"iv_rank","op":"gt","value":55}]},"exit":{"operator":"OR","conditions":[{"field":"pnl_pct","op":"gte","value":50}]},"leg_selection":{"short_put":{"option_type":"put","delta_target":0.25,"dte_min":25,"dte_max":45,"side":"sell","position_intent":"sell_to_open","ratio":1},"long_put":{"option_type":"put","delta_target":0.10,"dte_min":25,"dte_max":45,"side":"buy","position_intent":"buy_to_open","ratio":1}},"position_sizing":{"method":"max_risk","max_risk_usd":1000},"management":{"close_at_profit_pct":50,"close_at_dte":7,"roll_at_dte":0,"stop_loss_pct":100}}` diff --git a/internal/discovery/options/orchestrator.go b/internal/discovery/options/orchestrator.go index fedb7a83..f17f5154 100644 --- a/internal/discovery/options/orchestrator.go +++ b/internal/discovery/options/orchestrator.go @@ -239,10 +239,19 @@ func RunOptionsDiscovery(ctx context.Context, cfg OptionsDiscoveryConfig, deps O } if !cfg.DryRun { - if createErr := deps.Strategies.Create(ctx, &strategy); createErr != nil { + createdStrategy, created, createErr := discovery.CreateOrReusePaperStrategy(ctx, deps.Strategies, strategy) + if createErr != nil { result.Errors = append(result.Errors, fmt.Sprintf("deploy %s: %v", w.ticker, createErr)) continue } + strategy = createdStrategy + if !created { + logger.Info("options/discovery: strategy already exists, reusing", + slog.String("id", strategy.ID.String()), + slog.String("ticker", strategy.Ticker), + slog.String("name", strategy.Name), + ) + } } result.Winners = append(result.Winners, OptionsDeployedStrategy{ diff --git a/internal/discovery/orchestrator.go b/internal/discovery/orchestrator.go index daa2c48a..f65cabd2 100644 --- a/internal/discovery/orchestrator.go +++ b/internal/discovery/orchestrator.go @@ -274,10 +274,19 @@ func RunDiscovery(ctx context.Context, cfg DiscoveryConfig, deps DiscoveryDeps) } if !cfg.DryRun { - if createErr := deps.Strategies.Create(ctx, &strategy); createErr != nil { + createdStrategy, created, createErr := CreateOrReusePaperStrategy(ctx, deps.Strategies, strategy) + if createErr != nil { result.Errors = append(result.Errors, fmt.Sprintf("deploy %s: %v", strategy.Ticker, createErr)) continue } + strategy = createdStrategy + if !created { + logger.Info("discovery: strategy already exists, reusing", + slog.String("id", strategy.ID.String()), + slog.String("ticker", strategy.Ticker), + slog.String("name", strategy.Name), + ) + } } deployed := DeployedStrategy{ diff --git a/internal/domain/trade.go b/internal/domain/trade.go index 74655d31..92a72d1f 100644 --- a/internal/domain/trade.go +++ b/internal/domain/trade.go @@ -11,6 +11,7 @@ type Trade struct { ID uuid.UUID `json:"id"` OrderID *uuid.UUID `json:"order_id,omitempty"` PositionID *uuid.UUID `json:"position_id,omitempty"` + ExternalID string `json:"external_id,omitempty"` Ticker string `json:"ticker"` Side OrderSide `json:"side"` Quantity float64 `json:"quantity"` diff --git a/internal/integration/testhelpers_test.go b/internal/integration/testhelpers_test.go index e162e7ca..4b06a38e 100644 --- a/internal/integration/testhelpers_test.go +++ b/internal/integration/testhelpers_test.go @@ -226,6 +226,7 @@ func applyDDL(t *testing.T, pool *pgxpool.Pool) { // Trades `CREATE TABLE trades ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_id TEXT, order_id UUID REFERENCES orders (id), position_id UUID REFERENCES positions (id), ticker TEXT NOT NULL, diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 56b5d002..83c3f1da 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -22,6 +22,7 @@ type Metrics struct { SignalParseFailuresTotal prometheus.Counter SchedulerTickTotal *prometheus.CounterVec AutomationJobErrorsTotal *prometheus.CounterVec + AlpacaReconcileRunsTotal *prometheus.CounterVec StaleRunsReconciled prometheus.Counter PortfolioValue prometheus.Gauge PositionsOpen prometheus.Gauge @@ -107,6 +108,11 @@ func New() *Metrics { Help: "Total automation job errors by job name.", }, []string{"job_name"}), + AlpacaReconcileRunsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "tradingagent_alpaca_reconcile_runs_total", + Help: "Total Alpaca reconciliation runs by outcome.", + }, []string{"result"}), + StaleRunsReconciled: prometheus.NewCounter(prometheus.CounterOpts{ Name: "tradingagent_stale_runs_reconciled_total", Help: "Total number of stale pipeline runs force-failed by the reconciler.", @@ -172,6 +178,7 @@ func New() *Metrics { m.SignalParseFailuresTotal, m.SchedulerTickTotal, m.AutomationJobErrorsTotal, + m.AlpacaReconcileRunsTotal, m.StaleRunsReconciled, m.PortfolioValue, m.PositionsOpen, @@ -242,6 +249,13 @@ func (m *Metrics) RecordAutomationJobError(jobName string) { m.AutomationJobErrorsTotal.WithLabelValues(jobName).Inc() } +func (m *Metrics) RecordAlpacaReconcileRun(result string) { + if m == nil { + return + } + m.AlpacaReconcileRunsTotal.WithLabelValues(result).Inc() +} + func (m *Metrics) RecordStaleRunReconciled() { m.StaleRunsReconciled.Inc() } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index c8ef258e..050731d0 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -237,6 +237,20 @@ tradingagent_scheduler_tick_total{type="strategy"} 1 # TYPE tradingagent_automation_job_errors_total counter tradingagent_automation_job_errors_total{job_name="reconcile_orders"} 1 tradingagent_automation_job_errors_total{job_name="sync_positions"} 2 +`, + }, + { + name: "alpaca reconcile runs", + collector: func(m *metrics.Metrics) prometheus.Collector { return m.AlpacaReconcileRunsTotal }, + add: func(m *metrics.Metrics) { + m.RecordAlpacaReconcileRun("success") + m.RecordAlpacaReconcileRun("success") + m.RecordAlpacaReconcileRun("error") + }, + want: `# HELP tradingagent_alpaca_reconcile_runs_total Total Alpaca reconciliation runs by outcome. +# TYPE tradingagent_alpaca_reconcile_runs_total counter +tradingagent_alpaca_reconcile_runs_total{result="error"} 1 +tradingagent_alpaca_reconcile_runs_total{result="success"} 2 `, }, { diff --git a/internal/repository/postgres/order_test.go b/internal/repository/postgres/order_test.go index 61ed1a24..76d6533c 100644 --- a/internal/repository/postgres/order_test.go +++ b/internal/repository/postgres/order_test.go @@ -390,6 +390,7 @@ func newOrderTradeIntegrationPool(t *testing.T, ctx context.Context) (*pgxpool.P )`, `CREATE TABLE trades ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_id TEXT, order_id UUID REFERENCES orders (id), position_id UUID REFERENCES positions (id), ticker TEXT NOT NULL, diff --git a/internal/repository/postgres/schema_version.go b/internal/repository/postgres/schema_version.go index c7c45cba..3ee044ab 100644 --- a/internal/repository/postgres/schema_version.go +++ b/internal/repository/postgres/schema_version.go @@ -9,7 +9,7 @@ import ( ) // RequiredSchemaVersion is the minimum schema version this runtime requires. -const RequiredSchemaVersion = 30 +const RequiredSchemaVersion = 32 type schemaVersionState string diff --git a/internal/repository/postgres/schema_version_test.go b/internal/repository/postgres/schema_version_test.go index dc4ce8bd..f28f4eab 100644 --- a/internal/repository/postgres/schema_version_test.go +++ b/internal/repository/postgres/schema_version_test.go @@ -45,7 +45,7 @@ func TestCompareSchemaVersion(t *testing.T) { }{ {name: "behind", current: 28, required: RequiredSchemaVersion, want: schemaVersionBehind}, {name: "match", current: RequiredSchemaVersion, required: RequiredSchemaVersion, want: schemaVersionMatch}, - {name: "ahead", current: 31, required: RequiredSchemaVersion, want: schemaVersionAhead}, + {name: "ahead", current: RequiredSchemaVersion + 1, required: RequiredSchemaVersion, want: schemaVersionAhead}, } for _, tt := range tests { diff --git a/internal/repository/postgres/strategy_test.go b/internal/repository/postgres/strategy_test.go index 40145537..b57fdf66 100644 --- a/internal/repository/postgres/strategy_test.go +++ b/internal/repository/postgres/strategy_test.go @@ -223,6 +223,41 @@ func TestStrategyRepoIntegration_CreateListAndUpdateStatus(t *testing.T) { } } +func TestStrategyRepoIntegration_DiscoveryDuplicateRejectedByUniqueIndex(t *testing.T) { + ctx := context.Background() + pool, cleanup := newStrategyIntegrationPool(t, ctx) + defer cleanup() + + repo := NewStrategyRepo(pool) + + first := &domain.Strategy{ + Name: "discovery: PBM RSI Momentum Breakout", + Ticker: "PBM", + MarketType: domain.MarketTypeStock, + Status: domain.StrategyStatusActive, + IsPaper: true, + } + if err := repo.Create(ctx, first); err != nil { + t.Fatalf("Create(first) error = %v", err) + } + + duplicate := &domain.Strategy{ + Name: "discovery: PBM RSI Momentum Breakout", + Ticker: "PBM", + MarketType: domain.MarketTypeStock, + Status: domain.StrategyStatusActive, + IsPaper: true, + } + err := repo.Create(ctx, duplicate) + if err == nil { + t.Fatal("Create(duplicate) error = nil, want unique violation") + } + errText := strings.ToLower(err.Error()) + if !strings.Contains(errText, "unique") && !strings.Contains(errText, "duplicate") { + t.Fatalf("Create(duplicate) error = %v, want unique/duplicate violation", err) + } +} + // assertContains fails if substr is not found in s. func assertContains(t *testing.T, s, substr string) { t.Helper() @@ -286,7 +321,7 @@ func newStrategyIntegrationPool(t *testing.T, ctx context.Context) (*pgxpool.Poo } ddl := []string{ - `CREATE TYPE market_type AS ENUM ('stock', 'crypto', 'polymarket')`, + `CREATE TYPE market_type AS ENUM ('stock', 'crypto', 'polymarket', 'options')`, `CREATE TABLE strategies ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), name TEXT NOT NULL, @@ -301,6 +336,10 @@ func newStrategyIntegrationPool(t *testing.T, ctx context.Context) (*pgxpool.Poo created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() )`, + `CREATE UNIQUE INDEX idx_strategies_discovery_unique + ON strategies (ticker, market_type, is_paper, name) + WHERE is_paper = true + AND (name LIKE 'discovery:%' OR name LIKE 'options:%')`, } for _, stmt := range ddl { diff --git a/internal/repository/postgres/trade.go b/internal/repository/postgres/trade.go index 49cc652f..a9e5e998 100644 --- a/internal/repository/postgres/trade.go +++ b/internal/repository/postgres/trade.go @@ -30,10 +30,11 @@ func NewTradeRepo(pool *pgxpool.Pool) *TradeRepo { func (r *TradeRepo) Create(ctx context.Context, trade *domain.Trade) error { row := r.pool.QueryRow(ctx, `INSERT INTO trades ( - order_id, position_id, ticker, side, quantity, price, fee, executed_at + external_id, order_id, position_id, ticker, side, quantity, price, fee, executed_at ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING id, created_at`, + nullString(trade.ExternalID), trade.OrderID, trade.PositionID, trade.Ticker, @@ -71,7 +72,7 @@ func (r *TradeRepo) GetByPosition(ctx context.Context, positionID uuid.UUID, fil return r.list(ctx, query, args, "get trades by position") } -const tradeSelectSQL = `SELECT id, order_id, position_id, ticker, side, +const tradeSelectSQL = `SELECT id, external_id, order_id, position_id, ticker, side, quantity::double precision, price::double precision, fee::double precision, executed_at, created_at FROM trades` @@ -105,12 +106,14 @@ func (r *TradeRepo) list(ctx context.Context, query string, args []any, op strin func scanTrade(sc scanner) (*domain.Trade, error) { var ( trade domain.Trade + externalID *string orderID *uuid.UUID positionID *uuid.UUID ) err := sc.Scan( &trade.ID, + &externalID, &orderID, &positionID, &trade.Ticker, @@ -127,6 +130,9 @@ func scanTrade(sc scanner) (*domain.Trade, error) { trade.OrderID = orderID trade.PositionID = positionID + if externalID != nil { + trade.ExternalID = *externalID + } return &trade, nil } diff --git a/internal/repository/postgres/trade_test.go b/internal/repository/postgres/trade_test.go index cbd249cc..54a5ef42 100644 --- a/internal/repository/postgres/trade_test.go +++ b/internal/repository/postgres/trade_test.go @@ -81,6 +81,7 @@ func TestTradeRepoIntegration_CreateListGetByOrderAndPosition(t *testing.T) { tradeA := &domain.Trade{ OrderID: &orderID, PositionID: &positionID, + ExternalID: "fill-1", Ticker: "AAPL", Side: domain.OrderSideBuy, Quantity: 5, @@ -91,6 +92,7 @@ func TestTradeRepoIntegration_CreateListGetByOrderAndPosition(t *testing.T) { tradeB := &domain.Trade{ OrderID: &orderID, PositionID: &positionID, + ExternalID: "fill-2", Ticker: "AAPL", Side: domain.OrderSideBuy, Quantity: 5, @@ -101,6 +103,7 @@ func TestTradeRepoIntegration_CreateListGetByOrderAndPosition(t *testing.T) { tradeC := &domain.Trade{ OrderID: &orderID, PositionID: &otherPositionID, + ExternalID: "fill-3", Ticker: "MSFT", Side: domain.OrderSideSell, Quantity: 2, @@ -134,6 +137,9 @@ func TestTradeRepoIntegration_CreateListGetByOrderAndPosition(t *testing.T) { if byOrder[0].OrderID == nil || *byOrder[0].OrderID != orderID { t.Fatalf("expected returned trades to link to order %s, got %v", orderID, byOrder[0].OrderID) } + if byOrder[0].ExternalID == "" { + t.Fatal("expected returned trade to include external_id") + } byPosition, err := tradeRepo.GetByPosition(ctx, positionID, repository.TradeFilter{ StartDate: timePtr(baseTime.Add(1 * time.Minute)), diff --git a/migrations/000031_discovery_strategy_dedup.down.sql b/migrations/000031_discovery_strategy_dedup.down.sql new file mode 100644 index 00000000..9d45e410 --- /dev/null +++ b/migrations/000031_discovery_strategy_dedup.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_strategies_discovery_unique; diff --git a/migrations/000031_discovery_strategy_dedup.up.sql b/migrations/000031_discovery_strategy_dedup.up.sql new file mode 100644 index 00000000..f4e4fa77 --- /dev/null +++ b/migrations/000031_discovery_strategy_dedup.up.sql @@ -0,0 +1,60 @@ +-- Collapse existing duplicates for discovery-generated paper strategies, +-- preserving the earliest created row for each logical strategy key. +CREATE TEMP TABLE _strategy_dedup_map ON COMMIT DROP AS +WITH ranked AS ( + SELECT id, + FIRST_VALUE(id) OVER ( + PARTITION BY ticker, market_type, is_paper, name + ORDER BY created_at ASC, id ASC + ) AS keeper_id, + ROW_NUMBER() OVER ( + PARTITION BY ticker, market_type, is_paper, name + ORDER BY created_at ASC, id ASC + ) AS rn + FROM strategies + WHERE is_paper = true + AND (name LIKE 'discovery:%' OR name LIKE 'options:%') +) +SELECT id AS duplicate_id, keeper_id + FROM ranked + WHERE rn > 1; + +UPDATE backtest_configs bc + SET strategy_id = d.keeper_id + FROM _strategy_dedup_map d + WHERE bc.strategy_id = d.duplicate_id; + +UPDATE orders o + SET strategy_id = d.keeper_id + FROM _strategy_dedup_map d + WHERE o.strategy_id = d.duplicate_id; + +UPDATE positions p + SET strategy_id = d.keeper_id + FROM _strategy_dedup_map d + WHERE p.strategy_id = d.duplicate_id; + +UPDATE report_artifacts ra + SET strategy_id = d.keeper_id + FROM _strategy_dedup_map d + WHERE ra.strategy_id = d.duplicate_id; + +UPDATE pipeline_runs pr + SET strategy_id = d.keeper_id + FROM _strategy_dedup_map d + WHERE pr.strategy_id = d.duplicate_id; + +UPDATE agent_events ae + SET strategy_id = d.keeper_id + FROM _strategy_dedup_map d + WHERE ae.strategy_id = d.duplicate_id; + +DELETE FROM strategies s + USING _strategy_dedup_map d + WHERE s.id = d.duplicate_id; + +-- Enforce idempotency for discovery deploy runs. +CREATE UNIQUE INDEX IF NOT EXISTS idx_strategies_discovery_unique + ON strategies (ticker, market_type, is_paper, name) + WHERE is_paper = true + AND (name LIKE 'discovery:%' OR name LIKE 'options:%'); diff --git a/migrations/000032_trade_external_id.down.sql b/migrations/000032_trade_external_id.down.sql new file mode 100644 index 00000000..31dfcf0a --- /dev/null +++ b/migrations/000032_trade_external_id.down.sql @@ -0,0 +1,6 @@ +-- 000032_trade_external_id.down.sql + +DROP INDEX IF EXISTS idx_trades_external_id; + +ALTER TABLE trades + DROP COLUMN IF EXISTS external_id; diff --git a/migrations/000032_trade_external_id.up.sql b/migrations/000032_trade_external_id.up.sql new file mode 100644 index 00000000..38b571ff --- /dev/null +++ b/migrations/000032_trade_external_id.up.sql @@ -0,0 +1,7 @@ +-- 000032_trade_external_id.up.sql +-- Persist broker fill activity identifiers on trades for deterministic reconciliation dedupe. + +ALTER TABLE trades + ADD COLUMN IF NOT EXISTS external_id TEXT; + +CREATE INDEX IF NOT EXISTS idx_trades_external_id ON trades (external_id); diff --git a/migrations/discovery_strategy_dedup_migration_test.go b/migrations/discovery_strategy_dedup_migration_test.go new file mode 100644 index 00000000..4bfcf8dd --- /dev/null +++ b/migrations/discovery_strategy_dedup_migration_test.go @@ -0,0 +1,214 @@ +package migrations_test + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestDiscoveryStrategyDedupUpMigrationDefinesExpectedSQL(t *testing.T) { + upSQL := normalizeSQL(t, readMigrationFile(t, "000031_discovery_strategy_dedup.up.sql")) + + for _, fragment := range []string{ + "create temp table _strategy_dedup_map on commit drop as", + "partition by ticker, market_type, is_paper, name", + "where is_paper = true and (name like 'discovery:%' or name like 'options:%')", + "update backtest_configs", + "update orders", + "update positions", + "update report_artifacts", + "update pipeline_runs", + "update agent_events", + "delete from strategies", + "create unique index if not exists idx_strategies_discovery_unique", + "on strategies (ticker, market_type, is_paper, name)", + } { + if !strings.Contains(upSQL, fragment) { + t.Fatalf("expected up migration to contain %q, got:\n%s", fragment, upSQL) + } + } +} + +func TestDiscoveryStrategyDedupDownMigrationDropsIndex(t *testing.T) { + downSQL := normalizeSQL(t, readMigrationFile(t, "000031_discovery_strategy_dedup.down.sql")) + if !strings.Contains(downSQL, "drop index if exists idx_strategies_discovery_unique") { + t.Fatalf("expected down migration to drop discovery dedup index, got:\n%s", downSQL) + } +} + +func TestDiscoveryStrategyDedupMigrationAppliesAndEnforcesUniqueness(t *testing.T) { + if testing.Short() { + t.Skip("skipping migration integration test in short mode") + } + + databaseURL := os.Getenv("DB_URL") + if databaseURL == "" { + databaseURL = os.Getenv("DATABASE_URL") + } + if databaseURL == "" { + t.Skip("skipping migration integration test: DB_URL or DATABASE_URL is not set") + } + + ctx := context.Background() + adminPool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("failed to create admin pool: %v", err) + } + t.Cleanup(adminPool.Close) + + if _, err := adminPool.Exec(ctx, `CREATE EXTENSION IF NOT EXISTS pgcrypto`); err != nil { + t.Fatalf("failed to ensure pgcrypto extension: %v", err) + } + + schemaName := "migr_" + strings.ReplaceAll(uuid.NewString(), "-", "") + sanitizedSchemaName := pgx.Identifier{schemaName}.Sanitize() + if _, err := adminPool.Exec(ctx, `CREATE SCHEMA `+sanitizedSchemaName); err != nil { + t.Fatalf("failed to create schema: %v", err) + } + t.Cleanup(func() { + if _, err := adminPool.Exec(ctx, `DROP SCHEMA IF EXISTS `+sanitizedSchemaName+` CASCADE`); err != nil { + t.Errorf("failed to drop schema %q: %v", schemaName, err) + } + }) + + config, err := pgxpool.ParseConfig(databaseURL) + if err != nil { + t.Fatalf("failed to parse database config: %v", err) + } + config.ConnConfig.RuntimeParams["search_path"] = schemaName + ",public" + config.ConnConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol + + pool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + t.Fatalf("failed to create schema-scoped pool: %v", err) + } + t.Cleanup(pool.Close) + + for _, filename := range sortedUpMigrationsThrough(t, "000030_embeddings.up.sql") { + if _, err := pool.Exec(ctx, readMigrationFile(t, filename)); err != nil { + t.Fatalf("failed to apply %s: %v", filename, err) + } + } + + // Seed duplicates before migration 000031 runs. + var keeperStockID uuid.UUID + if err := pool.QueryRow(ctx, ` +INSERT INTO strategies (name, ticker, market_type, is_paper, status) +VALUES ('discovery: PBM RSI Momentum Breakout', 'PBM', 'stock', true, 'active') +RETURNING id +`).Scan(&keeperStockID); err != nil { + t.Fatalf("failed to seed keeper stock strategy: %v", err) + } + for i := 0; i < 2; i++ { + if _, err := pool.Exec(ctx, ` +INSERT INTO strategies (name, ticker, market_type, is_paper, status) +VALUES ('discovery: PBM RSI Momentum Breakout', 'PBM', 'stock', true, 'active') +`); err != nil { + t.Fatalf("failed to seed stock duplicate %d: %v", i, err) + } + } + for i := 0; i < 2; i++ { + if _, err := pool.Exec(ctx, ` +INSERT INTO strategies (name, ticker, market_type, is_paper, status) +VALUES ('options: QQQ bull_put_spread', 'QQQ', 'options', true, 'active') +`); err != nil { + t.Fatalf("failed to seed options duplicate %d: %v", i, err) + } + } + + if _, err := pool.Exec(ctx, ` +INSERT INTO backtest_configs (strategy_id, name, start_date, end_date, simulation_params) +VALUES ($1, 'dedup cfg', DATE '2025-01-01', DATE '2025-02-01', '{}'::jsonb) +`, keeperStockID); err != nil { + t.Fatalf("failed to seed backtest config: %v", err) + } + + // Attach one backtest config to a duplicate strategy id so migration must rewire it. + var duplicateStockID uuid.UUID + if err := pool.QueryRow(ctx, ` +SELECT id + FROM strategies + WHERE name = 'discovery: PBM RSI Momentum Breakout' + AND ticker = 'PBM' + AND market_type = 'stock' + AND is_paper = true + AND id <> $1 + ORDER BY created_at DESC, id DESC + LIMIT 1 +`, keeperStockID).Scan(&duplicateStockID); err != nil { + t.Fatalf("failed to select duplicate stock strategy id: %v", err) + } + if _, err := pool.Exec(ctx, ` +UPDATE backtest_configs + SET strategy_id = $2 + WHERE strategy_id = $1 +`, keeperStockID, duplicateStockID); err != nil { + t.Fatalf("failed to point backtest config at duplicate strategy: %v", err) + } + + if _, err := pool.Exec(ctx, readMigrationFile(t, "000031_discovery_strategy_dedup.up.sql")); err != nil { + t.Fatalf("failed to apply 000031 up migration: %v", err) + } + + var stockCount int + if err := pool.QueryRow(ctx, ` +SELECT COUNT(*) FROM strategies + WHERE name = 'discovery: PBM RSI Momentum Breakout' + AND ticker = 'PBM' + AND market_type = 'stock' + AND is_paper = true +`).Scan(&stockCount); err != nil { + t.Fatalf("failed counting deduped stock strategies: %v", err) + } + if stockCount != 1 { + t.Fatalf("stock dedup count = %d, want 1", stockCount) + } + + var optionsCount int + if err := pool.QueryRow(ctx, ` +SELECT COUNT(*) FROM strategies + WHERE name = 'options: QQQ bull_put_spread' + AND ticker = 'QQQ' + AND market_type = 'options' + AND is_paper = true +`).Scan(&optionsCount); err != nil { + t.Fatalf("failed counting deduped options strategies: %v", err) + } + if optionsCount != 1 { + t.Fatalf("options dedup count = %d, want 1", optionsCount) + } + + var rewiredCount int + if err := pool.QueryRow(ctx, ` +SELECT COUNT(*) + FROM backtest_configs + WHERE strategy_id = $1 +`, keeperStockID).Scan(&rewiredCount); err != nil { + t.Fatalf("failed counting rewired backtest configs: %v", err) + } + if rewiredCount != 1 { + t.Fatalf("rewired backtest config count = %d, want 1", rewiredCount) + } + + // Verify uniqueness is now enforced by index. + _, err = pool.Exec(ctx, ` +INSERT INTO strategies (name, ticker, market_type, is_paper, status) +VALUES ('discovery: PBM RSI Momentum Breakout', 'PBM', 'stock', true, 'active') +`) + if err == nil { + t.Fatal("expected unique violation after dedup migration, got nil") + } + errText := strings.ToLower(err.Error()) + if !strings.Contains(errText, "unique") && !strings.Contains(errText, "duplicate") { + t.Fatalf("expected unique/duplicate error, got: %v", err) + } + + if _, err := pool.Exec(ctx, readMigrationFile(t, "000031_discovery_strategy_dedup.down.sql")); err != nil { + t.Fatalf("failed to apply 000031 down migration: %v", err) + } +} diff --git a/web/src/components/universe/watchlist-table.test.tsx b/web/src/components/universe/watchlist-table.test.tsx index a3b97ecb..e504d471 100644 --- a/web/src/components/universe/watchlist-table.test.tsx +++ b/web/src/components/universe/watchlist-table.test.tsx @@ -47,12 +47,33 @@ describe('WatchlistTable', () => { expect(screen.getByText('gap_up')).toBeInTheDocument() }) - it('renders 0.00 when score is undefined (null-guard)', () => { + it('falls back to watch_score when score is absent', () => { render( , + { wrapper: Wrapper }, + ) + + expect(screen.getByText('MSFT')).toBeInTheDocument() + expect(screen.getByText('0.64')).toBeInTheDocument() + }) + + it('renders 0.00 when score is undefined (null-guard)', () => { + render( + { { wrapper: Wrapper }, ) - expect(screen.getByText('MSFT')).toBeInTheDocument() + expect(screen.getByText('NVDA')).toBeInTheDocument() expect(screen.getByText('0.00')).toBeInTheDocument() }) diff --git a/web/src/components/universe/watchlist-table.tsx b/web/src/components/universe/watchlist-table.tsx index d75de9cc..6da8dd36 100644 --- a/web/src/components/universe/watchlist-table.tsx +++ b/web/src/components/universe/watchlist-table.tsx @@ -1,10 +1,12 @@ import { useNavigate } from 'react-router-dom' import { Badge } from '@/components/ui/badge' -import type { ScoredTicker } from '@/lib/api/types' +import type { ScoredTicker, TrackedTicker } from '@/lib/api/types' + +type WatchlistTicker = ScoredTicker | TrackedTicker interface WatchlistTableProps { - tickers: ScoredTicker[] + tickers: WatchlistTicker[] } function scoreBadgeVariant(score: number | undefined) { @@ -13,6 +15,36 @@ function scoreBadgeVariant(score: number | undefined) { return 'destructive' as const } +function resolveTickerScore(ticker: WatchlistTicker): number { + if ('score' in ticker && typeof ticker.score === 'number') { + return ticker.score + } + if ('watch_score' in ticker && typeof ticker.watch_score === 'number') { + return ticker.watch_score + } + return 0 +} + +function resolveTickerChangePct(ticker: WatchlistTicker): number { + return 'change_pct' in ticker && typeof ticker.change_pct === 'number' ? ticker.change_pct : 0 +} + +function resolveTickerGapPct(ticker: WatchlistTicker): number { + return 'gap_pct' in ticker && typeof ticker.gap_pct === 'number' ? ticker.gap_pct : 0 +} + +function resolveTickerDayVolume(ticker: WatchlistTicker): number { + return 'day_volume' in ticker && typeof ticker.day_volume === 'number' ? ticker.day_volume : 0 +} + +function resolveTickerDayClose(ticker: WatchlistTicker): number { + return 'day_close' in ticker && typeof ticker.day_close === 'number' ? ticker.day_close : 0 +} + +function resolveTickerReasons(ticker: WatchlistTicker): string[] { + return 'reasons' in ticker && Array.isArray(ticker.reasons) ? ticker.reasons : [] +} + export function WatchlistTable({ tickers }: WatchlistTableProps) { const navigate = useNavigate() @@ -35,53 +67,58 @@ export function WatchlistTable({ tickers }: WatchlistTableProps) { - {tickers.map((t) => ( - - navigate(`/discovery?tickers=${encodeURIComponent(t.ticker)}`) - } - > - {t.ticker} - - - {(t.score ?? 0).toFixed(2)} - - - = 0 ? 'text-emerald-400' : 'text-red-400' - }`} - > - {(t.change_pct ?? 0) >= 0 ? '+' : ''} - {(t.change_pct ?? 0).toFixed(2)}% - - = 0 ? 'text-emerald-400' : 'text-red-400' - }`} + {tickers.map((t) => { + const score = resolveTickerScore(t) + const changePct = resolveTickerChangePct(t) + const gapPct = resolveTickerGapPct(t) + const dayVolume = resolveTickerDayVolume(t) + const dayClose = resolveTickerDayClose(t) + const reasons = resolveTickerReasons(t) + + return ( + navigate(`/discovery?tickers=${encodeURIComponent(t.ticker)}`)} > - {(t.gap_pct ?? 0) >= 0 ? '+' : ''} - {(t.gap_pct ?? 0).toFixed(2)}% - - - {formatVolume(t.day_volume ?? 0)} - - - ${(t.day_close ?? 0).toFixed(2)} - - -
- {(t.reasons ?? []).map((reason, i) => ( - - {reason} - - ))} -
- - - ))} + {t.ticker} + + {score.toFixed(2)} + + = 0 ? 'text-emerald-400' : 'text-red-400' + }`} + > + {changePct >= 0 ? '+' : ''} + {changePct.toFixed(2)}% + + = 0 ? 'text-emerald-400' : 'text-red-400' + }`} + > + {gapPct >= 0 ? '+' : ''} + {gapPct.toFixed(2)}% + + + {formatVolume(dayVolume)} + + + ${dayClose.toFixed(2)} + + +
+ {reasons.map((reason, i) => ( + + {reason} + + ))} +
+ + + ) + })} diff --git a/web/src/lib/api/client.ts b/web/src/lib/api/client.ts index 481e3751..9988aa4b 100644 --- a/web/src/lib/api/client.ts +++ b/web/src/lib/api/client.ts @@ -488,7 +488,9 @@ export class ApiClient { } async getWatchlist(top: number = 30) { - return this.request('/api/v1/universe/watchlist', { query: { top } }); + return this.request>('/api/v1/universe/watchlist', { + query: { top }, + }); } async refreshUniverse() { diff --git a/web/src/pages/realtime-page.tsx b/web/src/pages/realtime-page.tsx index 1659e140..21c4965d 100644 --- a/web/src/pages/realtime-page.tsx +++ b/web/src/pages/realtime-page.tsx @@ -901,6 +901,7 @@ export function RealtimePage() { {/* Feed panel — second on mobile, first (left) on xl */}
+

Event feed

diff --git a/web/src/pages/universe-page.tsx b/web/src/pages/universe-page.tsx index 9877fb72..40d34351 100644 --- a/web/src/pages/universe-page.tsx +++ b/web/src/pages/universe-page.tsx @@ -53,7 +53,7 @@ export function UniversePage() { }) const tickers: TrackedTicker[] = universeQuery.data?.data ?? [] - const watchlist: ScoredTicker[] = watchlistQuery.data ?? [] + const watchlist = (watchlistQuery.data ?? []) as Array return (