diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 88bff6d..498cb2d 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -8,8 +8,7 @@ on: permissions: contents: read - # Optional: allow read access to pull request. Use with `only-new-issues` option. - # pull-requests: read + pull-requests: read jobs: golangci: @@ -22,5 +21,5 @@ jobs: go-version: stable - name: golangci-lint uses: golangci/golangci-lint-action@v9 -# with: -# version: v1.64 + with: + version: latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..18d448f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: Go Tests + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.26.2' + cache: true + + - name: Install Dependencies + run: go mod download + + - name: Run Tests + run: go test -race -v ./... \ No newline at end of file diff --git a/.gitignore b/.gitignore index 33b29c1..cd5c94f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,16 @@ config.yaml config.yml !config.example.yaml +dev-config.yaml +*.pdf +*.html +e2e-tests* +main +go.work +go.work.sum +vendor +*.txt +notes.md +.claude +AGENTS.md +CLAUDE.md \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..fb81e0f --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,45 @@ +version: "2" + +run: + timeout: 5m + tests: true + +linters: + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + - path: _test\.go + linters: + - errcheck + - govet + - staticcheck + - unused + - revive + - gocritic + - gocyclo + - dupl + - funlen + - goconst + - lll + - misspell + - unparam + - unconvert + - gosec + - gochecknoinits + - gochecknoglobals + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 + +output: + formats: + text: + path: stdout + print-linter-name: true + print-issued-lines: true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0f9b128 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# Changelog + +All notable changes to this project will be documented in this file. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7f64022 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Build stage +FROM golang:1.26-alpine AS builder + +WORKDIR /app + +# Install build dependencies +RUN apk add --no-cache git ca-certificates + +# Copy go mod files and vendor directory first for better layer caching +COPY go.mod go.sum ./ +COPY vendor/ ./vendor/ + +# Copy source code +COPY . . + +# Build the binary using vendor dependencies +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -mod=vendor -ldflags="-w -s" -o /futureq ./cmd/futureq + +# Runtime stage +FROM alpine:3.24 + +WORKDIR /app + +# Install ca-certificates for HTTPS and runas non-root user +RUN apk add --no-cache ca-certificates tzdata && \ + adduser -D -u 1000 appuser + +USER appuser + +# Copy binary from builder +COPY --from=builder /futureq /app/futureq + +# Expose default gRPC port (can be overridden via config) +EXPOSE 50051 + +ENTRYPOINT ["/app/futureq"] +CMD ["start"] diff --git a/README.md b/README.md index e7f91d6..46c143a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,180 @@ # FutureQ +**A high-performance, distributed delayed-message queue broker written in Go.** +FutureQ lets producers publish messages with a relative delay and guarantees reliable dispatch to consumers when the delay expires. It combines durable embedded storage, Raft-based replication, and bidirectional gRPC streaming into a single, easy-to-operate binary. + +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) +[![Go](https://img.shields.io/badge/Go-1.26-00ADD8?logo=go)](go.mod) + +--- + +## Features + +- **Delayed messaging** — enqueue a message with a `delay_ms`; it becomes visible to consumers only after the delay expires. +- **Durable storage** — disk-backed by [Pebble](https://github.com/cockroachdb/pebble) (CockroachDB's LSM store) with a time-optimized key schema, or pure in-memory for ephemeral workloads. +- **High availability** — multi-node replication via [Dragonboat](https://github.com/lni/dragonboat) (multi-group Raft), with a metadata Raft group for cluster membership. +- **Dynamic membership** — nodes join and leave a running cluster over gRPC (`JoinCluster` / `LeaveCluster`); no static bootstrap list required after the first node. +- **Consumer groups & topics** — topic-based routing with fan-out across groups and round-robin dispatch within a group. +- **At-least-once delivery** — in-flight tracking with automatic re-dispatch of unacknowledged messages; batched deletes amortize LSM tombstone costs. +- **Message TTL** — a background janitor removes expired messages that were never consumed. +- **Observability** — Prometheus metrics endpoint and structured logging (zap) out of the box. + +## Technology Stack + +| Concern | Choice | +| ------------ | --------------------------------------------- | +| Language | Go 1.26 | +| Storage | Pebble (LSM tree) | +| Consensus | Dragonboat (multi-group Raft) | +| Transport | gRPC (bidirectional streaming) | +| Metrics | Prometheus | +| Protocol | [`futureq-io/protocol`](https://github.com/futureq-io/protocol) (Protobuf) | + +## Architecture Overview + +``` + ┌──────────────┐ PublishStream ┌──────────────────────────┐ + │ Producer │ ─────────────────▶│ │ + └──────────────┘ │ FutureQ Node │ + │ │ + ┌──────────────┐ Subscribe │ ┌────────────────────┐ │ + │ Consumer │ ◀──────────────── │ │ gRPC API (8443) │ │ + └──────────────┘ │ └─────────┬──────────┘ │ + │ ▼ │ + ┌──────────────┐ Raft (50005) │ ┌────────────────────┐ │ + │ Other Nodes │ ◀───────────────▶ │ │ Dispatcher / Hub │ │ + └──────────────┘ │ │ Deleter · Janitor │ │ + │ └─────────┬──────────┘ │ + ┌──────────────┐ Prometheus │ ▼ │ + │ Metrics │ ◀── (9090) ──────│ │ Pebble + Raft log │ │ + └──────────────┘ │ └────────────────────┘ │ + └──────────────────────────┘ +``` + +- **Writes** go to the Raft leader (or straight to Pebble in standalone mode) and are stored under time-bucketed keys for efficient expiry scans. +- **Reads** are push-based: a dispatcher continuously scans for matured messages and routes them to connected consumers through a hub using a round-robin strategy. +- **Acks** are batched by a deleter and committed as a single Raft proposal, keeping write amplification low. + +Delivery is **at-least-once** — consumers should be idempotent. + +## Quick Start + +### Prerequisites + +- Go 1.26+ +- (Optional) Docker + +### Build & run a standalone node + +```bash +git clone https://github.com/futureq-io/futureq.git +cd futureq +go build -o futureq ./cmd/futureq + +cp config.example.yaml config.yaml # adjust as needed +./futureq start -c config.yaml +``` + +A standalone node (no `raft` section, or `raft.enabled: false`) writes directly to Pebble — perfect for local development. + +### Run a 3-node cluster + +On the first node, enable Raft and list all initial members: + +```yaml +raft: + enabled: true + nodeId: 1 + clusterId: 1 + listenAddress: "0.0.0.0:50005" + initialMembers: + 1: "10.0.0.1:50005" +``` + +Additional nodes join dynamically — no need to edit `initialMembers`: + +```bash +./futureq start -c node2.yaml --join 10.0.0.1:8443 +./futureq start -c node3.yaml --join 10.0.0.1:8443 +``` + +On first start the node contacts each seed until one accepts its `JoinCluster` request; membership is registered on both the event shard and the metadata group. Restarts detect local Raft data and skip the join flow automatically. + +### Docker + +```bash +docker build -t futureq . +docker run -p 8443:8443 -p 9090:9090 -p 50005:50005 \ + -v $(pwd)/config.yaml:/app/config.yaml \ + futureq start -c /app/config.yaml +``` + +## Configuration + +Every value is documented in [`config.example.yaml`](config.example.yaml), which mirrors the built-in defaults. Key sections: + +| Section | Highlights | +| ---------------- | ----------------------------------------------------------------- | +| `server` | gRPC listen address, connection limits, message size caps | +| `storage` | Engine (`pebble`), persistence toggle, time-bucket granularity | +| `storage.pebble` | WAL toggle, data path, cache/memtable sizing | +| `raft` | Node/cluster IDs, listen address, initial members, snapshot tuning | +| `consumer` | Dispatch poll interval, batched-delete interval, in-flight timeout, TTL janitor interval | +| `observability` | Log level, Prometheus listen address | + +Every value can be overridden with environment variables using the `FUTUREQ_` prefix, replacing dots with underscores: + +```bash +export FUTUREQ_STORAGE_PEBBLE_DATAPATH="/var/lib/futureq/data" +export FUTUREQ_OBSERVABILITY_LOGGER_LEVEL="debug" +``` + +## API + +FutureQ speaks gRPC; protobuf definitions live in [`futureq-io/protocol`](https://github.com/futureq-io/protocol). + +| RPC | Type | Description | +| ------------------ | ------------------- | -------------------------------------------------- | +| `PublishStream` | bidi streaming | Publish batches of delayed messages; receive per-batch acks | +| `Subscribe` | bidi streaming | Receive messages for a topic/consumer group; ack over the same stream | +| `GetClusterInfo` | unary | Cluster topology, leader and member metadata | +| `JoinCluster` | unary | Add a node to the event shard and metadata group | +| `LeaveCluster` | unary | Gracefully remove a node from the cluster | +| `LeaveMetadata` | unary | Remove a node from the metadata group only | + +Metrics are exposed at `observability.metrics.addr` (default `:9090`) in Prometheus format. + +## Project Layout + +``` +internal/ + main.go # entrypoint + cmd/ # Cobra CLI (start, leave) + app/ # wiring: storage, repositories, Raft lifecycle + api/grpc/ # gRPC server + handlers (producer, consumer, cluster) + dispatcher/ # scan/dispatch loop, hub, deleter, TTL janitor + storage/ # Pebble engine, time-bucket key schema + raft/ # Dragonboat state machine & event commands + repository/ # event repository abstraction + config/ # config loading + env overrides + metrics/ # Prometheus server +pkg/ + raft/metadata/ # metadata-group Raft (cluster membership) + log/ # zap logger setup + utils/ # shared helpers +``` + +## Development + +```bash +go build ./... # build +go test ./... # run tests +golangci-lint run # lint +``` + +Contributions are welcome — please open an issue to discuss substantial changes before sending a PR. + +## License + +[MIT](LICENSE) diff --git a/cmd/futureq/leave.go b/cmd/futureq/leave.go new file mode 100644 index 0000000..c31419e --- /dev/null +++ b/cmd/futureq/leave.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + stdLogger "log" + "time" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/futureq-io/futureq/internal/config" + pb "github.com/futureq-io/protocol/proto/go" +) + +var leaveSeedAddr string + +// leaveCmd represents the leave command +var leaveCmd = &cobra.Command{ + Use: "leave", + Short: "Gracefully remove a node from the FutureQ Raft cluster", + Long: `Gracefully remove a node from the FutureQ cluster. + +The node is removed from the Raft group's voting membership. After leaving, +the node's Raft data can be safely deleted. + +Example: + futureq leave --seed localhost:8443 --config node2.yaml`, + Run: leaveRun, +} + +func init() { + leaveCmd.Flags().StringVar(&leaveSeedAddr, "seed", "", "gRPC address of a seed node in the cluster") + _ = leaveCmd.MarkFlagRequired("seed") + + rootCmd.AddCommand(leaveCmd) +} + +func leaveRun(_ *cobra.Command, _ []string) { + cfg, err := config.Load(cfgFile) + if err != nil { + stdLogger.Fatalf("failed to load config: %v", err) + } + + if !cfg.Raft.Enabled { + stdLogger.Fatalf("raft must be enabled in config to leave a cluster") + } + + conn, err := grpc.NewClient(leaveSeedAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + stdLogger.Fatalf("failed to connect to seed node: %v", err) + } + defer conn.Close() //nolint:errcheck + + client := pb.NewFutureQClusterClient(conn) + + req := &pb.LeaveRequest{ + NodeId: cfg.Raft.NodeID, + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + stdLogger.Printf("Requesting node %d to leave cluster via seed %s...", cfg.Raft.NodeID, leaveSeedAddr) + resp, err := client.LeaveCluster(ctx, req) + if err != nil { + stdLogger.Fatalf("LeaveCluster RPC failed: %v", err) + } + + if !resp.Success { + stdLogger.Fatalf("failed to leave cluster: %s", resp.ErrorMessage) + } + + stdLogger.Printf("Node %d successfully left the cluster. Raft data can now be safely deleted.", cfg.Raft.NodeID) +} diff --git a/cmd/futureq/root.go b/cmd/futureq/root.go new file mode 100644 index 0000000..cd20a2a --- /dev/null +++ b/cmd/futureq/root.go @@ -0,0 +1,31 @@ +/* +Copyright © 2025 Ahmad Anvari +*/ +package main + +import ( + "os" + + "github.com/spf13/cobra" +) + +var cfgFile string + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "futureq", + Short: "FutureQ server", + Long: `FutureQ is a highly available distrubuted scheduled message queue`, +} + +func init() { + rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "Path to config file") + + rootCmd.AddCommand(startCmd) +} + +func main() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/cmd/futureq/start.go b/cmd/futureq/start.go new file mode 100644 index 0000000..240d2f5 --- /dev/null +++ b/cmd/futureq/start.go @@ -0,0 +1,228 @@ +/* +Copyright © 2025 FutureQ Authors +*/ +package main + +import ( + "context" + stdLogger "log" + "time" + + "github.com/spf13/cobra" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + grpcserver "github.com/futureq-io/futureq/internal/api/grpc" + "github.com/futureq-io/futureq/internal/app" + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/dispatcher" + "github.com/futureq-io/futureq/internal/metrics" + "github.com/futureq-io/futureq/pkg/log" + pb "github.com/futureq-io/protocol/proto/go" +) + +var joinSeeds []string + +// startCmd represents the server command +var startCmd = &cobra.Command{ + Use: "start", + Short: "Start the FutureQ broker", + Long: `Start the FutureQ broker. + +To join an existing cluster, pass one or more seed addresses: + futureq start -c node2.yaml --join 10.0.0.1:8443 --join 10.0.0.2:8443 + +On first start, the node contacts each seed in order until one accepts +its JoinCluster request. Membership is registered on both the event +shard and the metadata group. Subsequent restarts skip the join flow +automatically (local Raft data is detected).`, + Run: startRun, +} + +func init() { + startCmd.Flags().StringSliceVar(&joinSeeds, "join", nil, "gRPC addresses of seed nodes to join (repeatable)") +} + +func startRun(_ *cobra.Command, _ []string) { + cfg, err := config.Load(cfgFile) + if err != nil { + stdLogger.Fatalf("failed to load config: %v", err) + } + + logger, err := log.InitLogger(cfg.Observability.Logger) + if err != nil { + stdLogger.Fatalf("failed to init logger: %v", err) + } + + // ── Initialise app: storage + repository ─────────────────────────────────── + a, err := app.Init(cfg, logger) + if err != nil { + logger.Fatal("failed to init app", zap.Error(err)) + } + + if err := a.WithRepositories(); err != nil { + logger.Fatal("failed to init repositories", zap.Error(err)) + } + + // ── Join an existing cluster if requested ──────────────────────────────── + // Only performed on a fresh node (no local Raft data). Restarts detect + // the existing data and skip the join flow entirely. + joining := false + if cfg.Raft.Enabled && len(joinSeeds) > 0 { + if a.HasRaftData() { + logger.Info("local raft data found, skipping join flow") + } else { + joinCluster(cfg, joinSeeds, logger) + joining = true + } + } + + // ── Dispatcher components ───────────────────────────────────────────────── + wakeCh := make(chan struct{}, 1) + strategy := dispatcher.NewRoundRobinStrategy() + hub := dispatcher.NewHub(strategy, logger, wakeCh) + + inFlightTimeout := time.Duration(cfg.Consumer.InFlightTimeoutMs) * time.Millisecond + deleteInterval := time.Duration(cfg.Consumer.DeleteBatchIntervalMs) * time.Millisecond + dispatchInterval := time.Duration(cfg.Consumer.DispatchPollIntervalMs) * time.Millisecond + janitorInterval := time.Duration(cfg.Consumer.TTLJanitorIntervalMs) * time.Millisecond + + // ── Build the delete backend ──────────────────────────────────────────────── + // In Raft mode: route deletions through SyncPropose(DeleteBatchCmd). + // In standalone mode: write deletions directly to the local storage engine. + var deleteBackend dispatcher.DeleteBackend + if cfg.Raft.Enabled { + proposeDelete := func(cmd []byte) error { + ctx, cancel := context.WithTimeout(a.Ctx, 5*time.Second) + defer cancel() + session := a.NodeHost.GetNoOPSession(cfg.Raft.ClusterID) + _, err := a.NodeHost.SyncPropose(ctx, session, cmd) + return err + } + deleteBackend = dispatcher.NewRaftDeleteBackend(proposeDelete, logger) + } else { + deleteBackend = dispatcher.NewDirectDeleteBackend(a.DB, logger) + } + + deleter := dispatcher.NewDeleter(deleteBackend, deleteInterval, logger) + disp := dispatcher.NewDispatcher( + a.DB, hub, deleter, + dispatchInterval, inFlightTimeout, + wakeCh, logger, + ) + + // Wire the OnDelete callback so the deleter notifies the dispatcher when + // a delete completes — removes the key from the in-flight tracker. + deleter.OnDelete = func(key []byte) { + disp.RemoveInFlight(key) + } + + // ── Prometheus metrics server ────────────────────────────────────────────── + // Start before Raft so the liveness/readiness probes have something to hit + // while the Raft cluster is still forming. + metricsSrv := metrics.NewServer(cfg.Observability.Metrics.Addr, logger) + a.RegisterComponentWithShutdown() + go func() { + defer a.ComponentShutdownDone() + metricsSrv.Run(a.Ctx) + }() + + // ── Start Raft (must be after WithRepositories so the repo is ready) ────── + // onDeleteKeys is called by the state machine after each DeleteBatchCmd + // is committed. We wire it to the dispatcher so in-flight entries are + // removed immediately without waiting for the next scan pass. + if cfg.Raft.Enabled { + if err := a.StartRaft(joining, disp.RemoveInFlightBatch); err != nil { + logger.Fatal("failed to start raft", zap.Error(err)) + } + } + + // ── TTL Janitor ─────────────────────────────────────────────────────────── + janitor := dispatcher.NewTTLJanitor(a.DB, deleter, janitorInterval, logger) + + // ── Start background goroutines ─────────────────────────────────────────── + a.RegisterComponentWithShutdown() + go func() { + defer a.ComponentShutdownDone() + deleter.Run(a.Ctx) + }() + + a.RegisterComponentWithShutdown() + go func() { + defer a.ComponentShutdownDone() + disp.Run(a.Ctx) + }() + + a.RegisterComponentWithShutdown() + go func() { + defer a.ComponentShutdownDone() + janitor.Run(a.Ctx) + }() + + // ── gRPC server ─────────────────────────────────────────────────────────── + grpcserver.New(cfg.Server, hub, deleter, logger). + Listen(). + WaitForShutdown(a.Ctx) + + // ── Block until SIGTERM / SIGINT ────────────────────────────────────────── + if err := a.WithGracefulShutdown(); err != nil { + logger.Fatal("failed to graceful shutdown", zap.Error(err)) + } +} + +// joinCluster contacts each seed in order until one accepts this node's +// JoinCluster request. Membership is registered on both the event shard +// and the metadata group by the seed. +func joinCluster(cfg *config.Config, seeds []string, logger *zap.Logger) { + req := &pb.JoinRequest{ + NodeId: cfg.Raft.NodeID, + RaftAddress: cfg.Raft.ListenAddress, + GrpcAddress: cfg.Server.Listen, + } + + for _, seed := range seeds { + logger.Info("attempting to join cluster via seed", + zap.String("seed", seed), + zap.Uint64("node_id", cfg.Raft.NodeID), + ) + + conn, err := grpc.NewClient(seed, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + logger.Warn("failed to connect to seed", zap.String("seed", seed), zap.Error(err)) + continue + } + + client := pb.NewFutureQClusterClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + resp, err := client.JoinCluster(ctx, req) + cancel() + _ = conn.Close() + + if err != nil { + logger.Warn("JoinCluster RPC failed", + zap.String("seed", seed), + zap.Error(err), + ) + continue + } + if !resp.Success { + logger.Warn("seed rejected join", + zap.String("seed", seed), + zap.String("error", resp.ErrorMessage), + ) + continue + } + + logger.Info("successfully joined cluster", + zap.String("seed", seed), + zap.Uint64("node_id", cfg.Raft.NodeID), + ) + return + } + + logger.Fatal("failed to join cluster: all seeds exhausted", + zap.Strings("seeds", seeds), + ) +} diff --git a/cmd/server.go b/cmd/server.go deleted file mode 100644 index 5eeb99d..0000000 --- a/cmd/server.go +++ /dev/null @@ -1,12 +0,0 @@ -/* -Copyright © 2025 NAME HERE -*/ -package main - -import ( - "github.com/futureq-io/futureq/internal/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/config.example.yaml b/config.example.yaml index 0f508c7..b2657c0 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,18 +1,131 @@ +# ============================================================================== +# Futureq Default Configuration Reference +# ============================================================================== +# This file represents the exact default state of a Futureq node. If you run +# the application without a config file, these are the values it will use. +# +# TO USE THIS FILE: +# 1. Copy it and rename it to `config.yaml`. +# 2. Adjust the values below to fit your deployment needs. +# 3. Start the node (e.g., `./futureq --config config.yaml`). +# +# ENVIRONMENT VARIABLE OVERRIDES: +# Every configuration value below can be overridden using environment variables +# by using the `FUTUREQ_` prefix and replacing dots with underscores. +# For example, to override the Pebble data path: +# export FUTUREQ_STORAGE_PEBBLE_DATAPATH="/var/lib/futureq/data" +# ============================================================================== + +server: + # Grpc listen address + listen: "0.0.0.0:8443" + maxConns: 10 + timeout: 5s + maxRecvSizeKb: 100 + maxSendSizeKb: 100 + observability: - logging: + logger: + # Controls the verbosity of the logs. + # Valid options: debug, info, warn, error, fatal level: info + metrics: + addr: "0.0.0.0:9090" + +storage: + type: "pebble" + minAckLevel: Quorum + + # When set to false, the node runs entirely in-memory using a virtual filesystem. + # All data will be destroyed when the process exits. Useful for testing or ephemeral workers. + persist: true + + # The bucket size for storing events. Minimum amount must be 0ms. (less than 1 millisecond is not supported) + # Using large time buckets (e.g., 1s) reduces the number of keys in the DB + # and may improve performance, but it also means you will have less precision on deliveries. + timeBucketSize: 1ms + + pebble: + # Disabling the Write-Ahead Log (WAL) increases write throughput but risks + # data loss of recently written keys in the event of a sudden crash. + disableWAL: false + + # The absolute or relative path where the database files will be stored on disk. + # Note: This is strictly ignored if `storage.persist` is set to false. + dataPath: "./data" + + # Size of the block cache in Megabytes. + # Increase this value to improve performance on read-heavy workloads. + cacheSizeMb: 16 + + # Size of the active in-memory table in Megabytes. + # Increase this value for write-heavy workloads. (Must be at least 1MB). + inMemoryTableSizeMb: 64 + + bolt: + dataPath: "./data" + defaultBucket: "futureq" + +raft: + enabled: false + # Unique integer ID of the current node in the Raft cluster (e.g. 1, 2, 3). + # Use '0' (or omit the raft section entirely) to run the node in standalone + # mode without Raft replication (writing directly to Pebble). + nodeId: 1 + + # Unique integer ID of the Raft replication shard/cluster. + clusterId: 1 + + # The address the Raft engine (Dragonboat NodeHost) listens on for + # intra-cluster consensus messages (heartbeats, proposals, snapshots). + listenAddress: "0.0.0.0:50005" + + # The directory where Raft WAL logs and cluster metadata are stored. + dataPath: "./raft-data" + + # Map of all initial voting members of the cluster (Node ID -> Raft Address). + # This list must be identical across all cluster nodes on initial bootstrap. + initialMembers: + 1: "0.0.0.0:50005" + + # Average round-trip latency (RTT) between Raft peers in milliseconds. + # Dragonboat uses this to calibrate election timeouts (10 * RTT) and + # heartbeat intervals (1 * RTT). + # + # - For production WAN/cloud deploys: use 100-200. + # - For low-latency data centers or local test runs: set to 10-20 to trigger + # faster leader failover and speed up the replication loop. + rttMillisecond: 200 + + # Number of committed Raft log entries to write before a new database + # snapshot is created and old log files are deleted (compacted) from disk. + # + # - Higher values (e.g. 100000) are recommended for high-throughput write + # workloads to avoid frequent I/O spikes and write stalls. + # - Lower values (e.g. 10000) keep the disk usage of Raft logs smaller and + # make node restarts faster. + snapShotEntries: 10000 + + # Number of log entries to retain after taking a snapshot. This acts as + # a buffer so that if a follower falls slightly behind, the leader can + # sync it using incremental log updates rather than a full snapshot stream. + compactionOverHead: 5000 + +consumer: + # How long (in milliseconds) the dispatcher sleeps between Pebble scan passes + # when no ready messages are found. Lower values reduce delivery latency at + # the cost of slightly more Pebble iterator overhead. + dispatchPollIntervalMs: 50 + + # How often (in milliseconds) the batched deleter flushes acknowledged message + # keys to Pebble as a single batch. Batching amortises the LSM tombstone write + # cost; individual key deletions after every ACK would be far more expensive. + deleteBatchIntervalMs: 500 + + # How long (in milliseconds) a dispatched-but-unacknowledged message is + # considered abandoned and eligible for re-dispatch. + inFlightTimeoutMs: 5000 -persistence: - path: "./data" - -rabbitmq: - rabbitmq_server: - host: "127.0.0.1" - port: 5672 - username: guest - password: guest - virtual_host: "" - rabbitmq_data_exchange: - consume_queue_name: "consume-queue" - declare_queue: false - produce_queue_name: "produce-queue" + # How often (in milliseconds) the TTL janitor performs a full Pebble scan + # to remove expired messages that were never consumed. + ttlJanitorIntervalMs: 60000 \ No newline at end of file diff --git a/go.mod b/go.mod index 8bb8a35..924939b 100644 --- a/go.mod +++ b/go.mod @@ -1,54 +1,88 @@ module github.com/futureq-io/futureq -go 1.24 +go 1.26.2 + +// replace github.com/lni/dragonboat/v4 => github.com/hertzcodes/dragonboat v4.0.0 require ( - github.com/cockroachdb/pebble v1.1.5 + github.com/cespare/xxhash/v2 v2.3.0 + github.com/cockroachdb/pebble/v2 v2.1.6 + github.com/futureq-io/protocol/proto/go v0.1.9 github.com/google/uuid v1.6.0 - github.com/rabbitmq/amqp091-go v1.10.0 - github.com/spf13/cobra v1.10.1 - github.com/spf13/viper v1.21.0 - go.uber.org/zap v1.27.0 - gopkg.in/yaml.v3 v3.0.1 + github.com/lni/dragonboat/v4 v4.0.0-20250723143628-076c7f6497dc + github.com/prometheus/client_golang v1.16.0 + github.com/spf13/cobra v1.10.2 + github.com/spf13/viper v1.4.0 + github.com/stretchr/testify v1.11.1 + go.etcd.io/bbolt v1.5.0 + go.uber.org/zap v1.28.0 + google.golang.org/grpc v1.81.1 + google.golang.org/protobuf v1.36.11 + gopkg.in/yaml.v2 v2.4.0 ) require ( - github.com/DataDog/zstd v1.4.5 // indirect + github.com/DataDog/zstd v1.5.7 // indirect + github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect + github.com/RaduBerinde/axisds v0.1.0 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect + github.com/VictoriaMetrics/metrics v1.18.1 // indirect + github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v0.0.0-20221207173255-0f086d933dac // indirect github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect + github.com/google/btree v1.0.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-immutable-radix v1.0.0 // indirect + github.com/hashicorp/go-msgpack v0.5.3 // indirect + github.com/hashicorp/go-multierror v1.0.0 // indirect + github.com/hashicorp/go-sockaddr v1.0.0 // indirect + github.com/hashicorp/golang-lru v0.5.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/memberlist v0.3.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/lni/goutils v1.4.0 // indirect + github.com/lni/vfs v0.2.1-0.20220616104132-8852fd867376 // indirect + github.com/magiconair/properties v1.8.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/miekg/dns v1.1.26 // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect + github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/pelletier/go-toml v1.2.0 // indirect + github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.15.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect - github.com/spf13/afero v1.15.0 // indirect - github.com/spf13/cast v1.10.0 // indirect + github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect + github.com/spf13/afero v1.1.2 // indirect + github.com/spf13/cast v1.3.0 // indirect + github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/subosito/gotenv v1.6.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + github.com/valyala/fastrand v1.1.0 // indirect + github.com/valyala/histogram v1.2.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.28.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c461330..94bbc85 100644 --- a/go.sum +++ b/go.sum @@ -1,154 +1,577 @@ -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8= +github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= +github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= +github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= +github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw= -github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= +github.com/cockroachdb/pebble v0.0.0-20221207173255-0f086d933dac h1:pwyQPbghSh6PC4MgXNvMZjf19LTugkIIPUSRzAD5LEE= +github.com/cockroachdb/pebble v0.0.0-20221207173255-0f086d933dac/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0= +github.com/cockroachdb/pebble/v2 v2.1.6 h1:GDo7Z2+LgFZ7LJLdLmBXhDeTVIwgSPGxIT15hE7vGqM= +github.com/cockroachdb/pebble/v2 v2.1.6/go.mod h1:Reo1RTniv1UjVTAu/Fv74y5i3kJ5gmVrPhO9UtFiKn8= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI= +github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/futureq-io/protocol/proto/go v0.1.9 h1:fMmZYi9xbbgxTHnBISChpasz2gOyDa3uEqXzfSZv+xc= +github.com/futureq-io/protocol/proto/go v0.1.9/go.mod h1:VGAbKxcBAXSQIzqlrePjPXlomId4vUnVD8kgJ82xaGs= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0smaXiJZCNnLrvVBqirQVreixayXezGc= +github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= +github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lni/dragonboat/v4 v4.0.0-20250723143628-076c7f6497dc h1:5KPn/Yn1COC7w2InNxubbp5tqEYhtNy2l/EOt103ZlE= +github.com/lni/dragonboat/v4 v4.0.0-20250723143628-076c7f6497dc/go.mod h1:X35iFANAy9OKDck7Edgi408jnSUwgaSyIbq/XZkcw7M= +github.com/lni/goutils v1.4.0 h1:e1tNN+4zsbTpNvhG5cxirkH9Pdz96QAZ2j6+5tmjvqg= +github.com/lni/goutils v1.4.0/go.mod h1:LIHvF0fflR+zyXUQFQOiHPpKANf3UIr7DFIv5CBPOoU= +github.com/lni/vfs v0.2.1-0.20220616104132-8852fd867376 h1:jX9CoRWNPwrZ2yY3RJFTSwa49qDQqtXglrCByGdQGZg= +github.com/lni/vfs v0.2.1-0.20220616104132-8852fd867376/go.mod h1:LOatfyR8Xeej1jbXybwYGVfCccR0u+BQRG9xg7BD7xo= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= -github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= -github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= -github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= -github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= -github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= -github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= -github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= -github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= +github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/api/grpc/handlers/cluster.go b/internal/api/grpc/handlers/cluster.go new file mode 100644 index 0000000..f6df3b2 --- /dev/null +++ b/internal/api/grpc/handlers/cluster.go @@ -0,0 +1,267 @@ +package handlers + +import ( + "context" + "fmt" + "time" + + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/futureq-io/futureq/internal/app" + "github.com/futureq-io/futureq/pkg/raft/metadata" + pb "github.com/futureq-io/protocol/proto/go" +) + +// ClusterHandler implements pb.FutureQClusterServer. +type ClusterHandler struct { + pb.UnimplementedFutureQClusterServer + logger *zap.Logger +} + +func NewClusterHandler(logger *zap.Logger) *ClusterHandler { + return &ClusterHandler{ + logger: logger.Named("cluster_handler"), + } +} + +// ─── Cluster Info ───────────────────────────────────────────────────────────── + +// GetClusterInfo returns the current cluster topology from the metadata +// state machine. Any node can respond — it does not need to be the leader. +// In standalone (non-Raft) mode, returns info about this single node. +func (h *ClusterHandler) GetClusterInfo(ctx context.Context, _ *pb.ClusterInfoRequest) (*pb.ClusterInfoResponse, error) { + if app.A.NodeHost == nil || app.A.MetadataSM == nil { + return nil, status.Error(codes.NotFound, "attempt to get cluster info on single mode node") + } + + shardID := app.A.Config().Raft.ClusterID + topo := app.A.MetadataSM.GetShardTopology(shardID) + if topo == nil { + return nil, status.Error(codes.Unavailable, "topology not yet available") + } + + resp := &pb.ClusterInfoResponse{ + LeaderNodeId: topo.LeaderID, + LeaderAddress: topo.LeaderAddr, // gRPC address of the leader + Nodes: make([]*pb.NodeInfo, 0, len(topo.Nodes)+len(topo.NonVotings)), + } + + // Prefer gRPC addresses (client-facing); fall back to Raft addresses + // for nodes that haven't registered yet. + for nodeID := range topo.Nodes { + addr := topo.GrpcAddrs[nodeID] + if addr == "" { + addr = topo.Nodes[nodeID] + } + resp.Nodes = append(resp.Nodes, &pb.NodeInfo{ + NodeId: nodeID, + Address: addr, + IsLeader: nodeID == topo.LeaderID, + IsAlive: true, + }) + } + + // for nodeID := range topo.NonVotings { + // addr := topo.GrpcAddrs[nodeID] + // if addr == "" { + // addr = topo.NonVotings[nodeID] + // } + // resp.Nodes = append(resp.Nodes, &pb.NodeInfo{ + // NodeId: nodeID, + // Address: addr, + // IsLeader: false, + // IsAlive: true, + // }) + // } + + return resp, nil +} + +// ─── Cluster Membership ───────────────────────────────────────────────────── + +// JoinCluster adds a new broker node to the cluster. The node is registered +// on BOTH the event shard and the metadata group: first as a non-voting +// member to sync state, then promoted to a full voting replica on both once +// it has caught up with the leader. +func (h *ClusterHandler) JoinCluster(ctx context.Context, req *pb.JoinRequest) (*pb.JoinResponse, error) { + if app.A.NodeHost == nil { + return nil, status.Error(codes.FailedPrecondition, "node is not running in raft mode") + } + + eventShard := app.A.Config().Raft.ClusterID + + h.logger.Info("adding node as non-voting member", + zap.Uint64("node_id", req.NodeId), + zap.String("raft_address", req.RaftAddress), + zap.Uint64("event_shard", eventShard), + ) + + // Step 1: Add as non-voting member on both groups (parallel-safe order: + // metadata first since it's small and syncs fast). + if err := app.A.NodeHost.SyncRequestAddNonVoting(ctx, metadata.MetadataShardID, req.NodeId, req.RaftAddress, 0); err != nil { + h.logger.Error("failed to add non-voting member to metadata group", zap.Error(err)) + return &pb.JoinResponse{Success: false, ErrorMessage: err.Error()}, nil + } + if err := app.A.NodeHost.SyncRequestAddNonVoting(ctx, eventShard, req.NodeId, req.RaftAddress, 0); err != nil { + h.logger.Error("failed to add non-voting member to event shard", zap.Error(err)) + return &pb.JoinResponse{Success: false, ErrorMessage: err.Error()}, nil + } + + // Step 2: Wait for the node to catch up on both groups. + if err := h.waitForCatchUp(ctx, eventShard, req.NodeId); err != nil { + h.logger.Error("node failed to catch up on event shard", + zap.Uint64("node_id", req.NodeId), + zap.Error(err), + ) + return &pb.JoinResponse{Success: false, ErrorMessage: fmt.Sprintf("node did not catch up: %v", err)}, nil + } + if err := h.waitForCatchUp(ctx, metadata.MetadataShardID, req.NodeId); err != nil { + h.logger.Error("node failed to catch up on metadata group", + zap.Uint64("node_id", req.NodeId), + zap.Error(err), + ) + return &pb.JoinResponse{Success: false, ErrorMessage: fmt.Sprintf("node did not catch up: %v", err)}, nil + } + + h.logger.Info("promoting non-voting member to voter on both groups", + zap.Uint64("node_id", req.NodeId), + ) + + // Step 3: Promote to voting member on both groups. + if err := app.A.NodeHost.SyncRequestAddReplica(ctx, metadata.MetadataShardID, req.NodeId, req.RaftAddress, 0); err != nil { + h.logger.Error("failed to promote to voter on metadata group", zap.Error(err)) + return &pb.JoinResponse{Success: false, ErrorMessage: err.Error()}, nil + } + if err := app.A.NodeHost.SyncRequestAddReplica(ctx, eventShard, req.NodeId, req.RaftAddress, 0); err != nil { + h.logger.Error("failed to promote to voter on event shard", zap.Error(err)) + return &pb.JoinResponse{Success: false, ErrorMessage: err.Error()}, nil + } + + h.logger.Info("successfully joined cluster", + zap.Uint64("node_id", req.NodeId), + ) + + return &pb.JoinResponse{Success: true}, nil +} + +// LeaveCluster removes a node from both the event shard and the metadata group. +func (h *ClusterHandler) LeaveCluster(ctx context.Context, req *pb.LeaveRequest) (*pb.LeaveResponse, error) { + if app.A.NodeHost == nil { + return nil, status.Error(codes.FailedPrecondition, "node is not running in raft mode") + } + + eventShard := app.A.Config().Raft.ClusterID + + h.logger.Info("removing node from cluster", + zap.Uint64("node_id", req.NodeId), + zap.Uint64("event_shard", eventShard), + ) + + if err := app.A.NodeHost.SyncRequestDeleteReplica(ctx, eventShard, req.NodeId, 0); err != nil { + h.logger.Error("failed to remove node from event shard", zap.Error(err)) + return &pb.LeaveResponse{Success: false, ErrorMessage: err.Error()}, nil + } + if err := app.A.NodeHost.SyncRequestDeleteReplica(ctx, metadata.MetadataShardID, req.NodeId, 0); err != nil { + h.logger.Error("failed to remove node from metadata group", zap.Error(err)) + return &pb.LeaveResponse{Success: false, ErrorMessage: err.Error()}, nil + } + + h.logger.Info("successfully removed node from cluster", + zap.Uint64("node_id", req.NodeId), + ) + + return &pb.LeaveResponse{Success: true}, nil +} + +// ─── Metadata Group Membership ─────────────────────────────────────────────── + +// JoinMetadata adds a non-voting observer to the metadata Raft group. +// Client SDKs and new broker nodes use this to receive real-time topology +// updates without participating in metadata consensus. +func (h *ClusterHandler) JoinMetadata(ctx context.Context, req *pb.JoinMetadataRequest) (*pb.JoinMetadataResponse, error) { + if app.A.NodeHost == nil { + return nil, status.Error(codes.FailedPrecondition, "node is not running in raft mode") + } + + h.logger.Info("adding observer to metadata group", + zap.Uint64("node_id", req.NodeId), + zap.String("raft_address", req.RaftAddress), + ) + + if err := app.A.NodeHost.SyncRequestAddNonVoting(ctx, metadata.MetadataShardID, req.NodeId, req.RaftAddress, 0); err != nil { + h.logger.Error("failed to add metadata observer", zap.Error(err)) + return &pb.JoinMetadataResponse{Success: false, ErrorMessage: err.Error()}, nil + } + + h.logger.Info("successfully added metadata observer", + zap.Uint64("node_id", req.NodeId), + ) + + return &pb.JoinMetadataResponse{Success: true}, nil +} + +// LeaveMetadata removes a non-voting observer from the metadata Raft group. +func (h *ClusterHandler) LeaveMetadata(ctx context.Context, req *pb.LeaveMetadataRequest) (*pb.LeaveMetadataResponse, error) { + if app.A.NodeHost == nil { + return nil, status.Error(codes.FailedPrecondition, "node is not running in raft mode") + } + + h.logger.Info("removing observer from metadata group", + zap.Uint64("node_id", req.NodeId), + ) + + if err := app.A.NodeHost.SyncRequestDeleteReplica(ctx, metadata.MetadataShardID, req.NodeId, 0); err != nil { + h.logger.Error("failed to remove metadata observer", zap.Error(err)) + return &pb.LeaveMetadataResponse{Success: false, ErrorMessage: err.Error()}, nil + } + + h.logger.Info("successfully removed metadata observer", + zap.Uint64("node_id", req.NodeId), + ) + + return &pb.LeaveMetadataResponse{Success: true}, nil +} + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +// waitForCatchUp polls until the given node appears in the shard membership +// and is ready to be promoted. It uses Dragonboat's membership API to verify +// the node has been added, then waits a short period for state sync. +func (h *ClusterHandler) waitForCatchUp(ctx context.Context, shardID, nodeID uint64) error { + deadline := time.After(30 * time.Second) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + h.logger.Info("waiting for node to catch up", + zap.Uint64("node_id", nodeID), + zap.Uint64("shard_id", shardID), + ) + + for { + select { + case <-deadline: + return fmt.Errorf("timeout waiting for node %d to catch up", nodeID) + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + membership, err := app.A.NodeHost.SyncGetShardMembership(ctx, shardID) + if err != nil { + h.logger.Debug("failed to get shard membership, retrying", + zap.Error(err), + ) + continue + } + + // Check if the node is in the non-voting member list. + if _, ok := membership.NonVotings[nodeID]; ok { + h.logger.Info("node caught up, ready for promotion", + zap.Uint64("node_id", nodeID), + zap.Uint64("config_change_id", membership.ConfigChangeID), + ) + return nil + } + } + } +} diff --git a/internal/api/grpc/handlers/consumer.go b/internal/api/grpc/handlers/consumer.go new file mode 100644 index 0000000..d4b1cc5 --- /dev/null +++ b/internal/api/grpc/handlers/consumer.go @@ -0,0 +1,230 @@ +package handlers + +import ( + "context" + "errors" + "io" + + "github.com/google/uuid" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/futureq-io/futureq/internal/app" + "github.com/futureq-io/futureq/internal/dispatcher" + "github.com/futureq-io/futureq/internal/metrics" + pb "github.com/futureq-io/protocol/proto/go" +) + +// ConsumerHandler implements pb.FutureQConsumerServer. +type ConsumerHandler struct { + pb.UnimplementedFutureQConsumerServer + logger *zap.Logger + hub *dispatcher.Hub + deleter *dispatcher.Deleter +} + +// NewConsumerHandler returns an initialised ConsumerHandler. +func NewConsumerHandler(logger *zap.Logger, hub *dispatcher.Hub, deleter *dispatcher.Deleter) *ConsumerHandler { + return &ConsumerHandler{ + logger: logger.Named("consumer"), + hub: hub, + deleter: deleter, + } +} + +// Subscribe handles a bidirectional stream where the server pushes QueueMessage +// items to the client and the client replies with ConsumerFrame (AckRequest). +// +// Protocol: +// 1. The client must send a ConsumerFrame with a SubscribeInit as the first frame. +// This declares the topic and optional consumer group for this connection. +// 2. All subsequent client frames must carry AckRequest. +// 3. The server pushes QueueMessage frames as messages become eligible. +// +// Group semantics: +// - Empty group_id: universal consumer — receives every message on the topic. +// - Non-empty group_id: competing consumer — races with other consumers in +// the same group; only one receives each message. +// +// Delivery semantics: at-least-once. +// - On ACK (success=true): the key is queued for Raft-replicated deletion. +// - On NACK (success=false): the key is immediately removed from in-flight, +// making the message eligible for re-dispatch on the next dispatcher tick. +func (h *ConsumerHandler) Subscribe(stream grpc.BidiStreamingServer[pb.ConsumerFrame, pb.QueueMessage]) error { + // ─── Read and validate the SubscribeInit handshake ───────────────────────── + init, err := h.readInit(stream) + if err != nil { + return err + } + + // ─── Verify leadership in Raft mode ──────────────────────────────────────── + // TODO: WE SHALL ENABLE CONSUME ON REPLICAS + if err := h.checkLeadership(init); err != nil { + return err + } + + // ─── Register consumer with the Hub ──────────────────────────────────────── + consumerID := uuid.New().String() + ch := make(chan *pb.QueueMessage, 1024) + h.hub.Register(consumerID, init.Topic, init.GroupId, ch) + + metrics.ActiveConsumers.WithLabelValues(init.Topic, init.GroupId).Inc() + defer func() { + h.hub.Unregister(consumerID) + metrics.ActiveConsumers.WithLabelValues(init.Topic, init.GroupId).Dec() + h.logger.Info("consumer disconnected", + zap.String("id", consumerID), + zap.String("topic", init.Topic), + zap.String("group_id", init.GroupId), + ) + }() + + h.logger.Info("consumer connected", + zap.String("id", consumerID), + zap.String("topic", init.Topic), + zap.String("group_id", init.GroupId), + zap.Bool("universal", init.GroupId == ""), + ) + + ctx, cancel := context.WithCancel(stream.Context()) + defer cancel() + + errCh := make(chan error, 2) + + // ─── Sender goroutine: push messages to the consumer ───────────────────── + go h.sender(ctx, stream, ch, errCh) + + // ─── Receiver goroutine: process ACK/NACK frames ───────────────────────── + go h.receiver(stream, consumerID, init, errCh) + + err = <-errCh + if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && err != io.EOF { + h.logger.Error("consumer stream ended with error", + zap.Error(err), + zap.String("id", consumerID), + zap.String("topic", init.Topic), + zap.String("group_id", init.GroupId), + ) + return status.Errorf(codes.Internal, "stream error: %v", err) + } + + return nil +} + +// readInit reads and validates the mandatory SubscribeInit first frame. +func (h *ConsumerHandler) readInit(stream grpc.BidiStreamingServer[pb.ConsumerFrame, pb.QueueMessage]) (*pb.SubscribeInit, error) { + initFrame, err := stream.Recv() + if err != nil { + if err == io.EOF { + return nil, nil + } + return nil, status.Errorf(codes.Internal, "failed to read init frame: %v", err) + } + + init := initFrame.GetInit() + if init == nil { + return nil, status.Errorf(codes.InvalidArgument, + "first frame must be a SubscribeInit; got %T", initFrame.Body) + } + if init.Topic == "" { + return nil, status.Errorf(codes.InvalidArgument, "SubscribeInit.topic must not be empty") + } + + // group_id may be empty — that registers a universal (fan-out) consumer. + + return init, nil +} + +// checkLeadership verifies this node is the Raft leader (if Raft is enabled). +func (h *ConsumerHandler) checkLeadership(init *pb.SubscribeInit) error { + if app.A.NodeHost == nil { + return nil + } + + shardID := app.A.Config().Raft.ClusterID + leaderID, _, valid, err := app.A.NodeHost.GetLeaderID(shardID) + isLeader := err == nil && valid && leaderID == app.A.Config().Raft.NodeID + + if !isLeader { + h.logger.Warn("rejecting consumer: not the leader", + zap.String("topic", init.Topic), + zap.String("group_id", init.GroupId), + ) + return status.Errorf(codes.FailedPrecondition, + "node is not the cluster leader") + } + + return nil +} + +// sender pushes messages from the consumer's channel to the gRPC stream. +func (h *ConsumerHandler) sender( + ctx context.Context, + stream grpc.BidiStreamingServer[pb.ConsumerFrame, pb.QueueMessage], + ch chan *pb.QueueMessage, + errCh chan error, +) { + for { + select { + case <-ctx.Done(): + errCh <- ctx.Err() + return + case msg := <-ch: + if err := stream.Send(msg); err != nil { + errCh <- err + return + } + } + } +} + +// receiver processes incoming ACK/NACK frames from the consumer. +func (h *ConsumerHandler) receiver( + stream grpc.BidiStreamingServer[pb.ConsumerFrame, pb.QueueMessage], + consumerID string, + init *pb.SubscribeInit, + errCh chan error, +) { + for { + frame, err := stream.Recv() + if err != nil { + if err == io.EOF { + errCh <- nil + } else { + errCh <- err + } + return + } + + ackReq := frame.GetAck() + if ackReq == nil { + h.logger.Warn("received unexpected SubscribeInit after handshake", + zap.String("consumer_id", consumerID)) + continue + } + + success := ackReq.Success + + metrics.ConsumerAckTotal.WithLabelValues( + init.Topic, init.GroupId, boolToStr(success), + ).Inc() + + h.hub.RemoveInFlightForConsumer(consumerID, ackReq.DeliveryTag) + if success { + // ACK: queue the key for Raft-replicated deletion. + h.deleter.MarkDeleted(ackReq.DeliveryTag) + } + // NACK: the key remains in storage; the dispatcher will re-deliver it. + // In-flight gauge was incremented at dispatch time in the hub. + metrics.MessagesInFlight.WithLabelValues(init.Topic, init.GroupId).Dec() + } +} + +func boolToStr(b bool) string { + if b { + return "true" + } + return "false" +} diff --git a/internal/api/grpc/handlers/producer.go b/internal/api/grpc/handlers/producer.go new file mode 100644 index 0000000..3de215b --- /dev/null +++ b/internal/api/grpc/handlers/producer.go @@ -0,0 +1,267 @@ +package handlers + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "time" + + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/futureq-io/futureq/internal/app" + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/metrics" + raft "github.com/futureq-io/futureq/internal/raft/event" + "github.com/futureq-io/futureq/internal/storage" + "github.com/futureq-io/futureq/pkg/utils" + pb "github.com/futureq-io/protocol/proto/go" + storagepb "github.com/futureq-io/protocol/proto/go/storage" + "google.golang.org/protobuf/proto" +) + +var errBatchSave = errors.New("failed to save batch") + +// ProducerHandler implements pb.FutureQProducerServer. +type ProducerHandler struct { + pb.UnimplementedFutureQProducerServer + logger *zap.Logger + timeBucketSize time.Duration +} + +// NewProducerHandler returns an initialised ProducerHandler. +// In Raft mode the handler never writes directly to Pebble; all writes go +// through SyncPropose → state machine → Pebble. +func NewProducerHandler(logger *zap.Logger) *ProducerHandler { + return &ProducerHandler{ + logger: logger.Named("producer"), + timeBucketSize: app.A.Config().Storage.TimeBucketSize, + } +} + +// PublishStream handles a bidirectional stream where clients send PublishBatch +// frames and receive PublishBatchAck responses. +// +// Each batch is written atomically as a single Raft log entry (in Raft mode) +// or a single Pebble batch (standalone mode). The ack_level field controls +// whether the broker waits for quorum commit (ACK_LEVEL_QUORUM, default) or +// returns immediately after leader write (ACK_LEVEL_LEADER). +func (ph *ProducerHandler) PublishStream(stream grpc.BidiStreamingServer[pb.PublishBatch, pb.PublishBatchAck]) error { + for { + batch, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + ph.logger.Error("failed to receive from stream", zap.Error(err)) + return status.Errorf(codes.Internal, "stream read error: %v", err) + } + + ackResp, streamErr := ph.processBatch(stream.Context(), batch) + if streamErr != nil { + return streamErr + } + + if err := stream.Send(ackResp); err != nil { + ph.logger.Error("failed to send PublishBatchAck", zap.Error(err)) + return status.Errorf(codes.Internal, "failed to send ack: %v", err) + } + } +} + +// processBatch handles one PublishBatch frame and returns the corresponding ack. +func (ph *ProducerHandler) processBatch(ctx context.Context, batch *pb.PublishBatch) (*pb.PublishBatchAck, error) { + if len(batch.Messages) == 0 { + return &pb.PublishBatchAck{}, nil + } + + ackLevel := batch.GetAckLevel() + topicLabel := batchTopicLabel(batch) + start := time.Now() + + if app.A.Config().Storage.MinAckLevel == config.Quorum && ackLevel == pb.AckLevel_ACK_LEVEL_NO_ACK { + metrics.PublishRequestsTotal.WithLabelValues(topicLabel, ackLevel.String(), "rejected").Inc() + return &pb.PublishBatchAck{Success: false}, status.Error(codes.InvalidArgument, "NO_ACK level is not allowed when MinAckLevel is Quorum") + } + + nowMs := time.Now().UnixMilli() + + var processErr error + if app.A.NodeHost != nil { + processErr = ph.processRaftBatch(ctx, batch, nowMs, ackLevel) + } else { + processErr = ph.processStandaloneBatch(batch, nowMs) + } + + if processErr != nil { + metrics.PublishRequestsTotal.WithLabelValues(topicLabel, ackLevel.String(), "error").Inc() + return &pb.PublishBatchAck{Success: false}, processErr + } + + msgCount := len(batch.Messages) + metrics.PublishRequestsTotal.WithLabelValues(topicLabel, ackLevel.String(), "success").Inc() + metrics.MessagesPublishedTotal.WithLabelValues(topicLabel, ackLevel.String()).Add(float64(msgCount)) + metrics.PublishBatchSize.WithLabelValues(topicLabel).Observe(float64(msgCount)) + metrics.PublishLatencyMs.WithLabelValues(topicLabel, ackLevel.String()).Observe(float64(time.Since(start).Milliseconds())) + + return &pb.PublishBatchAck{Success: true}, nil +} + +// TODO: this is shit. we need to know how many messagers are per topic +func batchTopicLabel(batch *pb.PublishBatch) string { + if len(batch.Messages) == 0 { + return "" + } + + topicSet := make(map[string]struct{}) + result := bytes.Buffer{} + for i, m := range batch.Messages { + if _, exists := topicSet[m.Topic]; !exists { + result.WriteString(m.Topic) + topicSet[m.Topic] = struct{}{} + + if i < len(batch.Messages)-1 { + result.WriteString("|") + } + } + } + + return result.String() +} + +// marshalMessages is the single marshal loop shared by both write paths. +// +// For each message it computes the routing metadata (bucket, topicHash), +// builds the StoredMessage, marshals it exactly once, and calls fn with the +// result. If fn returns an error the loop stops immediately and the error is +// returned. +// +// Neither path re-serialises the bytes: the Raft path embeds them verbatim in +// the command buffer; the standalone path writes them directly to the Pebble +// batch. +func (ph *ProducerHandler) marshalMessages( + batch *pb.PublishBatch, + nowMs int64, + fn func(data *storagepb.StoredMessage) error, +) error { + for i, msg := range batch.Messages { + if msg.DelayMs < 0 { + return fmt.Errorf("negative delays are not allowed! delay: %d, topic: %s message_idx: %d", msg.DelayMs, msg.Topic, i) + } + + if msg.TtlMs < 0 { + return fmt.Errorf("negative ttls are not allowed! ttl: %d, topic: %s message_idx: %d", msg.TtlMs, msg.Topic, i) + } + + stored := &storagepb.StoredMessage{ + Topic: msg.Topic, + Payload: msg.Payload, + EnqueuedAtUnixMs: nowMs, + DelayMs: msg.DelayMs, + TtlMs: msg.TtlMs, + Indexes: msg.Indexes, + } + + if err := fn(stored); err != nil { + return err + } + } + + return nil +} + +func (ph *ProducerHandler) processRaftBatch( + ctx context.Context, + batch *pb.PublishBatch, + nowMs int64, + ackLevel pb.AckLevel, +) error { + shardID := app.A.Config().Raft.ClusterID + + // Only the leader may propose. + leaderID, _, valid, errL := app.A.NodeHost.GetLeaderID(shardID) + if errL != nil || !valid || leaderID != app.A.Config().Raft.NodeID { + return errors.New("node is not the cluster leader") + } + + items := make([]raft.StoreBatchItem, 0, len(batch.Messages)) + if err := ph.marshalMessages(batch, nowMs, func(data *storagepb.StoredMessage) error { + dataBytes, err := proto.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal message: %w", err) + } + + raftItem := raft.StoreBatchItem{ + Bucket: utils.CalculateBucket(data.EnqueuedAtUnixMs+data.DelayMs, app.A.Config().Storage.TimeBucketSize), + TopicHash: utils.TopicHash(data.Topic), + Msg: dataBytes, + } + + for _, idx := range data.GetIndexes() { + idxBytes, err := proto.Marshal(idx) + if err != nil { + return fmt.Errorf("failed to marshal index: %w", err) + } + + raftItem.Indexes = append(raftItem.Indexes, idxBytes) + } + + items = append(items, raftItem) + return nil + }); err != nil { + return fmt.Errorf("failed to build store batch: %w", err) + } + + cmdBytes, err := raft.MarshalStoreBatchCmd(items) + if err != nil { + return fmt.Errorf("failed to marshal StoreBatchCmd: %w", err) + } + + start := time.Now() + var proposeErr error + + switch ackLevel { + case pb.AckLevel_ACK_LEVEL_NO_ACK: + session := app.A.NodeHost.GetNoOPSession(shardID) + _, proposeErr = app.A.NodeHost.Propose(session, cmdBytes, 5*time.Second) + default: + propCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + session := app.A.NodeHost.GetNoOPSession(shardID) + _, proposeErr = app.A.NodeHost.SyncPropose(propCtx, session, cmdBytes) + cancel() + } + + metrics.RaftProposeDurationMs.WithLabelValues(ackLevel.String()).Observe( + float64(time.Since(start).Milliseconds()), + ) + + if proposeErr != nil { + return fmt.Errorf("failed to do raft proposal: %w", proposeErr) + } + + return nil +} + +// processStandaloneBatch writes the batch directly to Pebble (non-Raft mode). +func (ph *ProducerHandler) processStandaloneBatch(batch *pb.PublishBatch, nowMs int64) error { + b := app.A.DB.NewBatch() + defer func() { _ = b.Close() }() + + if err := ph.marshalMessages(batch, nowMs, func(data *storagepb.StoredMessage) error { + _, err := app.A.Repositories.Events.StoreWithBatch(b, data) + return err + }); err != nil { + return err + } + + if err := b.Commit(storage.Sync); err != nil { + ph.logger.Error("failed to commit standalone batch", zap.Error(err)) + return errBatchSave + } + + return nil +} diff --git a/internal/api/grpc/handlers/producer_test.go b/internal/api/grpc/handlers/producer_test.go new file mode 100644 index 0000000..eaf8da8 --- /dev/null +++ b/internal/api/grpc/handlers/producer_test.go @@ -0,0 +1,105 @@ +package handlers + +import ( + "testing" + "time" + + "github.com/futureq-io/futureq/pkg/utils" +) + +// TestCalculateBucket verifies the new bucket-index semantics. +// +// CalculateBucket returns floor(unixMs / bucketSizeMs), i.e. the integer bucket +// index — not a millisecond-aligned boundary. This is intentional: the bucket +// is used as a lexicographic key prefix in Pebble; its absolute value is not +// meaningful to consumers. +// +// Mapping: +// - 17000ms / 1000ms = bucket 17 +// - 17001ms / 1000ms = bucket 17 (same bucket as 17000) +// - 17999ms / 1000ms = bucket 17 (still bucket 17) +// - 18000ms / 1000ms = bucket 18 +// - 1500ms / 500ms = bucket 3 +// - 1501ms / 500ms = bucket 3 +// +// The dispatcher scans all keys with bucket <= currentBucket so messages +// are dispatched as soon as their bucket index is reached. +func TestCalculateBucket(t *testing.T) { + tests := []struct { + name string + executeAt int64 + bucketSize time.Duration + expected uint64 + }{ + { + name: "exact multiple of 1s", + executeAt: 17000, + bucketSize: 1 * time.Second, + expected: 17, // 17000 / 1000 = 17 + }, + { + name: "slightly over bucket boundary", + executeAt: 17001, + bucketSize: 1 * time.Second, + expected: 17, // 17001 / 1000 = 17 (floor division) + }, + { + name: "just under next bucket boundary", + executeAt: 17999, + bucketSize: 1 * time.Second, + expected: 17, // 17999 / 1000 = 17 (floor division) + }, + { + name: "exactly 0", + executeAt: 0, + bucketSize: 1 * time.Second, + expected: 0, + }, + { + name: "negative value treated as 0 bucket", + executeAt: -100, + bucketSize: 1 * time.Second, + expected: 0, // negative → bucket 0 (earliest possible) + }, + { + name: "bucket size is 0 (raw ms)", + executeAt: 17300, + bucketSize: 0, + expected: 17300, // bucketSize=0 → return raw ms as bucket + }, + { + name: "bucket size is 500ms, exact multiple", + executeAt: 1500, + bucketSize: 500 * time.Millisecond, + expected: 3, // 1500 / 500 = 3 + }, + { + name: "bucket size is 500ms, one ms over", + executeAt: 1501, + bucketSize: 500 * time.Millisecond, + expected: 3, // 1501 / 500 = 3 (floor division) + }, + { + name: "bucket size is 500ms, just under next boundary", + executeAt: 1999, + bucketSize: 500 * time.Millisecond, + expected: 3, // 1999 / 500 = 3 + }, + { + name: "bucket size is 500ms, at next boundary", + executeAt: 2000, + bucketSize: 500 * time.Millisecond, + expected: 4, // 2000 / 500 = 4 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := utils.CalculateBucket(tt.executeAt, tt.bucketSize) + if got != tt.expected { + t.Errorf("CalculateBucket(%d, %v) = %d; want %d", + tt.executeAt, tt.bucketSize, got, tt.expected) + } + }) + } +} diff --git a/internal/api/grpc/setup.go b/internal/api/grpc/setup.go new file mode 100644 index 0000000..139ad19 --- /dev/null +++ b/internal/api/grpc/setup.go @@ -0,0 +1,115 @@ +package grpc + +import ( + "context" + "net" + "time" + + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" + + "github.com/futureq-io/futureq/internal/api/grpc/handlers" + "github.com/futureq-io/futureq/internal/app" + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/dispatcher" + pb "github.com/futureq-io/protocol/proto/go" +) + +// Server wraps a *grpc.Server and exposes lifecycle methods. +type Server struct { + srv *grpc.Server + logger *zap.Logger + addr string +} + +// New creates a fully configured gRPC server and registers all service +// handlers. No network socket is opened yet; call Listen to do that. +func New( + cfg config.Server, + hub *dispatcher.Hub, + deleter *dispatcher.Deleter, + logger *zap.Logger, +) *Server { + log := logger.Named("grpc_server") + + srv := grpc.NewServer( + grpc.MaxConcurrentStreams(cfg.MaxConns), + grpc.MaxRecvMsgSize(cfg.MaxRecvSizeKB*1024), // KB + grpc.MaxSendMsgSize(cfg.MaxSendSizeKB*1024), // KB + + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 5 * time.Second, + PermitWithoutStream: true, + }), + + grpc.KeepaliveParams(keepalive.ServerParameters{ + MaxConnectionIdle: 30 * time.Second, + MaxConnectionAge: 2 * time.Minute, + MaxConnectionAgeGrace: 10 * time.Second, + Time: 10 * time.Second, + Timeout: cfg.Timeout, + }), + ) + + // Register all service implementations. + pb.RegisterFutureQProducerServer(srv, handlers.NewProducerHandler(log)) + pb.RegisterFutureQConsumerServer(srv, handlers.NewConsumerHandler(log, hub, deleter)) + pb.RegisterFutureQClusterServer(srv, handlers.NewClusterHandler(log)) + + return &Server{ + srv: srv, + logger: log, + addr: cfg.Listen, + } +} + +// Listen binds the TCP listener and blocks serving until the underlying +// gRPC server is stopped. Call Shutdown to stop it gracefully. +func (s *Server) Listen() *Server { + go func() { + lis, err := net.Listen("tcp", s.addr) + if err != nil { + s.logger.Fatal("gRPC: failed to bind", zap.String("address", s.addr), zap.Error(err)) + } + + s.logger.Info("gRPC server listening", zap.String("address", s.addr)) + + if err := s.srv.Serve(lis); err != nil { + s.logger.Fatal("gRPC: failed to serve", zap.Error(err)) + } + }() + + return s +} + +// WaitForShutdown registers a background shutdown handler that runs when ctx +// (the global app.Ctx) is cancelled. It gracefully stops the gRPC server +// within the deadline carried by app.A.ShutCtx (or a 10s fallback). +func (s *Server) WaitForShutdown(ctx context.Context) { + app.A.RegisterComponentWithShutdown() + + go func() { + defer app.A.ComponentShutdownDone() + + <-ctx.Done() + + s.logger.Info("gRPC server: initiating graceful shutdown") + + shutCtx := app.A.ShutCtx + + done := make(chan struct{}) + go func() { + s.srv.GracefulStop() + close(done) + }() + + select { + case <-done: + s.logger.Info("gRPC server: stopped gracefully") + case <-shutCtx.Done(): + s.logger.Warn("gRPC server: graceful shutdown timed out, forcing stop") + s.srv.Stop() + } + }() +} diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..000ed97 --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,375 @@ +package app + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "os/signal" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/lni/dragonboat/v4" + raftconfig "github.com/lni/dragonboat/v4/config" + "github.com/lni/dragonboat/v4/statemachine" + "go.uber.org/zap" + + "github.com/futureq-io/futureq/internal/config" + raft "github.com/futureq-io/futureq/internal/raft/event" + "github.com/futureq-io/futureq/pkg/raft/metadata" + "github.com/futureq-io/futureq/internal/repository" + "github.com/futureq-io/futureq/internal/storage" +) + +const gracefulShutdownTimeout = 10 * time.Second + +var A *App + +type Repositories struct { + Events *repository.EventRepository +} + +type App struct { + cfg *config.Config + DB storage.DB + NodeHost *dragonboat.NodeHost + Ctx context.Context + // ShutCtx is the 10-second shutdown window context. It is populated by + // WithGracefulShutdown immediately before a.Ctx is cancelled, so any + // goroutine watching a.Ctx.Done() can safely read ShutCtx. + ShutCtx context.Context + Repositories Repositories + cancel context.CancelCauseFunc + Logger *zap.Logger + wg sync.WaitGroup + + // MetadataSvc watches Dragonboat events and replicates topology changes + // through the metadata Raft group. Nil when Raft is disabled. + MetadataSvc *metadata.Service + + // MetadataSM is the in-memory metadata state machine instance. + // Provides direct read access to cluster topology. Nil when Raft is disabled. + MetadataSM *metadata.MetadataStateMachine +} + +// Init initialises the application: sets up Pebble storage and creates the App +// singleton. Call WithRepositories() next to load the EventRepository, then +// StartRaft() if clustering is enabled. +func Init(cfg *config.Config, logger *zap.Logger) (*App, error) { + a := &App{ + cfg: cfg, + Logger: logger.Named("app"), + } + + a.Ctx, a.cancel = context.WithCancelCause(context.Background()) + + var s storage.DB + var err error + switch cfg.Storage.Type { + case "pebble": + s, err = storage.NewPebble(cfg.Storage.Pebble, logger) + if err != nil { + return nil, fmt.Errorf("failed to initialize pebble storage: %w", err) + } + case "bolt": + s, err = storage.NewBoltDB(cfg.Storage.Bolt) + if err != nil { + return nil, fmt.Errorf("failed to initialize bolt storage: %w", err) + } + } + + a.DB = s + + A = a + + return a, nil +} + +// StartRaft starts the Dragonboat NodeHost and both Raft groups. +// +// Must be called after WithRepositories() so the EventRepository is fully +// initialised before the state machine factory captures it. +// +// Starts two Raft groups: +// 1. Event shard (config.Raft.ClusterID) — replicates event data +// 2. Metadata shard (metadata.MetadataShardID) — replicates cluster topology +// +// join controls Dragonboot bootstrap semantics: +// - false: bootstrap a new cluster using config.Raft.InitialMembers, or +// restart from local data when members are empty. +// - true: join an existing cluster as an already-registered member +// (initialMembers must be empty; membership was registered via JoinCluster). +// +// onDeleteKeys is called by the state machine after a DeleteBatchCmd is applied. +// Wire this to Dispatcher.RemoveInFlightBatch in start.go. +func (a *App) StartRaft(join bool, onDeleteKeys func(keys [][]byte)) error { + cfg := a.cfg + + // Create the metadata service first — it needs to be registered as the + // event listener before NodeHost is created. + var metadataSvc *metadata.Service + + nhc := raftconfig.NodeHostConfig{ + WALDir: cfg.Raft.DataPath, + NodeHostDir: cfg.Raft.DataPath, + RTTMillisecond: cfg.Raft.RTTMillisecond, + RaftAddress: cfg.Raft.ListenAddress, + // We'll set the listeners after creating the service below. + } + + // We need the NodeHost reference to create the propose function, but + // Dragonboat needs the listeners at creation time. Use a forward reference: + // create the service with a lazy propose function that captures nh later. + var nh *dragonboat.NodeHost + proposeMetadata := func(ctx context.Context, cmd []byte) error { + session := nh.GetNoOPSession(metadata.MetadataShardID) + _, err := nh.SyncPropose(ctx, session, cmd) + return err + } + + metadataSvc = metadata.NewService(nil, proposeMetadata, a.Logger) + nhc.RaftEventListener = metadataSvc + nhc.SystemEventListener = metadataSvc + + var err error + nh, err = dragonboat.NewNodeHost(nhc) + if err != nil { + return fmt.Errorf("failed to create dragonboat nodehost: %w", err) + } + + a.NodeHost = nh + a.MetadataSvc = metadataSvc + metadataSvc.SetNodeHost(nh) + + // Members are only passed when bootstrapping a brand-new cluster. + // Dragonboot semantics: + // - Fresh bootstrap: join=false + initialMembers populated + // - Fresh join: join=true + empty members (registered via JoinCluster) + // - Restart: join=false + empty members (local data exists) + members := make(map[uint64]dragonboat.Target) + if !join && !a.hasRaftData() { + for k, v := range cfg.Raft.InitialMembers { + members[k] = dragonboat.Target(v) + } + } + + // ── Start the metadata Raft group ────────────────────────────────────────── + // Wrap the factory to capture the state machine instance for direct reads. + var capturedSM *metadata.MetadataStateMachine + baseFactory := metadata.NewMetadataStateMachineFactory(a.Logger) + metadataFactory := func(clusterID, nodeID uint64) statemachine.IStateMachine { + sm := baseFactory(clusterID, nodeID) + if msm, ok := sm.(*metadata.MetadataStateMachine); ok { + capturedSM = msm + } + return sm + } + metadataRC := raftconfig.Config{ + ReplicaID: cfg.Raft.NodeID, + ShardID: metadata.MetadataShardID, + ElectionRTT: 10, + HeartbeatRTT: 1, + CheckQuorum: true, + SnapshotEntries: 5, + CompactionOverhead: 5, // compact aggressively since state is small + } + + if err := nh.StartReplica(members, join, metadataFactory, metadataRC); err != nil { + // Dragonboat panics internally on ErrShardNotBootstrapped (the "restarted + // during a previous bootstrap attempt" case). The panic is recovered by + // dragonboat's own handler and returned here as an error. When we hit it, + // the LogDB is in a half-initialised state that hasRaftData() would treat + // as "existing cluster data" on the next restart, causing an infinite + // crash loop. Wipe the dir so the next start begins from a clean slate. + if errors.Is(err, dragonboat.ErrShardNotBootstrapped) { + a.Logger.Warn("raft bootstrap failed (shard not bootstrapped); wiping raft data dir to allow clean retry", + zap.String("path", cfg.Raft.DataPath)) + _ = os.RemoveAll(cfg.Raft.DataPath) + } + return fmt.Errorf("failed to start metadata raft group: %w", err) + } + + a.MetadataSM = capturedSM + + // Wire the gRPC address registry into the metadata service so published + // topologies include client-facing addresses. + metadataSvc.SetGrpcAddrsSource(capturedSM.GetGrpcAddrs) + + // ── Start the event Raft group ───────────────────────────────────────────── + eventRC := raftconfig.Config{ + ReplicaID: cfg.Raft.NodeID, + ShardID: cfg.Raft.ClusterID, + ElectionRTT: 10, + HeartbeatRTT: 1, + CheckQuorum: true, + SnapshotEntries: cfg.Raft.SnapshotEntries, + CompactionOverhead: cfg.Raft.CompactionOverhead, + } + + // Pass the fully-initialised EventRepository so the state machine uses the + // same monotonic ID counter and key schema as the standalone write path. + eventFactory := raft.NewEventStateMachineFactory(a.DB, a.Repositories.Events, onDeleteKeys, a.Logger) + if err := nh.StartOnDiskReplica(members, join, eventFactory, eventRC); err != nil { + if errors.Is(err, dragonboat.ErrShardNotBootstrapped) { + a.Logger.Warn("event raft bootstrap failed (shard not bootstrapped); wiping raft data dir to allow clean retry", + zap.String("path", cfg.Raft.DataPath)) + _ = os.RemoveAll(cfg.Raft.DataPath) + } + return fmt.Errorf("failed to start event raft group: %w", err) + } + + // ── Announce our gRPC address to the cluster ────────────────────────────── + // Derive the advertise address from our Raft address (the identity the + // cluster knows us by) plus the gRPC port from Server.Listen. Both run + // in the same process, so the host is always the same. + grpcAdvertise, err := grpcAdvertiseAddr(nh.RaftAddress(), cfg.Server.Listen) + if err != nil { + return fmt.Errorf("failed to compute gRPC advertise address: %w", err) + } + // Retry until the metadata shard has a leader. During initial cluster + // bootstrap (OrderedReady), the first pod starts before peers exist, so + // the shard may not be ready yet. Keep retrying until it is. + for { + ctx, cancel := context.WithTimeout(a.Ctx, 5*time.Second) + err := metadataSvc.RegisterNodeAddr(ctx, cfg.Raft.NodeID, grpcAdvertise) + cancel() + if err == nil { + break + } + a.Logger.Warn("metadata shard not ready, retrying gRPC address registration", + zap.Error(err)) + select { + case <-a.Ctx.Done(): + return fmt.Errorf("failed to register gRPC address: %w", err) + case <-time.After(2 * time.Second): + } + } + + // Register the event shard with the metadata service so it publishes + // initial topology. + a.MetadataSvc.RegisterShard(cfg.Raft.ClusterID) + + return nil +} + +// grpcAdvertiseAddr computes the client-facing gRPC address for this node. +// The host is taken from raftAddr (this node's identity as the cluster sees +// it — always dialable by peers), and the port from grpcListen. +func grpcAdvertiseAddr(raftAddr, grpcListen string) (string, error) { + host, _, err := net.SplitHostPort(raftAddr) + if err != nil { + return "", fmt.Errorf("invalid raft address %q: %w", raftAddr, err) + } + _, grpcPort, err := net.SplitHostPort(grpcListen) + if err != nil { + return "", fmt.Errorf("invalid grpc listen address %q: %w", grpcListen, err) + } + return net.JoinHostPort(host, grpcPort), nil +} + +// HasRaftData reports whether local Raft data exists for this node. +// Used by the start command to distinguish a restart from a fresh join. +func (a *App) HasRaftData() bool { + return a.hasRaftData() +} + +// hasRaftData returns true if the Raft data directory contains a previously +// bootstrapped LogDB — meaning this node has been part of a cluster before. +// A bare directory (created by a failed first bootstrap attempt) is NOT +// treated as existing data; Dragonboat will retry the bootstrap from scratch. +func (a *App) hasRaftData() bool { + // Dragonboat's sharded-pebble LogDB stores each shard in a logdb-N + // subdirectory. The first shard writes a MANIFEST once initialised. + manifest := filepath.Join(a.cfg.Raft.DataPath, "logdb-0", "MANIFEST-000001") + _, err := os.Stat(manifest) + return err == nil +} + +// Config returns the application configuration. +func (a *App) Config() *config.Config { + return a.cfg +} + +// RegisterComponentWithShutdown increments the application wait group to track active components during shutdown. +func (a *App) RegisterComponentWithShutdown() { + a.wg.Add(1) +} + +// ComponentShutdownDone decrements the application wait group. +func (a *App) ComponentShutdownDone() { + a.wg.Done() +} + +func (a *App) WithGracefulShutdown() error { + sigterm := make(chan os.Signal, 1) + signal.Notify(sigterm, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigterm) + + <-sigterm + a.Logger.Info("received interrupt, shutting down gracefully...") + + // 1. Create the shared shutdown window. + // We MUST use context.Background() as the parent, because if we use a.Ctx, + // calling a.cancel() will immediately cancel shutCtx, causing an instant timeout. + shutCtx, shutCancel := context.WithTimeoutCause( + context.Background(), + gracefulShutdownTimeout, + errors.New("graceful shutdown timeout exceeded"), + ) + defer shutCancel() + + a.ShutCtx = shutCtx + + // 2. Signal all components to start winding down. + a.cancel(errors.New("graceful shutdown triggered")) + + // 3. Wait for all registered components to finish, or the timeout to expire. + waitDone := make(chan struct{}) + go func() { + a.wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + a.Logger.Info("graceful shutdown completed before timeout") + case <-shutCtx.Done(): + a.Logger.Warn("graceful shutdown timeout exceeded, forcing exit") + } + + if a.NodeHost != nil { + a.Logger.Info("closing Dragonboat NodeHost...") + a.NodeHost.Close() + a.Logger.Info("Dragonboat NodeHost closed successfully") + } + + if err := a.DB.Flush(); err != nil { + a.Logger.Error("failed to flush pebble on shutdown", zap.Error(err)) + } + + // 4. Safely close DB. + if a.DB != nil { + a.Logger.Info("closing DB...") + if err := a.DB.Close(); err != nil { + a.Logger.Error("failed to close DB", zap.Error(err)) + } else { + a.Logger.Info("DB closed successfully") + } + } + + return nil +} + +func (a *App) WithRepositories() error { + eventRepo, err := repository.NewEventRepository(a.DB, a.Logger, a.cfg.Storage.TimeBucketSize) + if err != nil { + return fmt.Errorf("failed to init event repo: %w", err) + } + + a.Repositories.Events = eventRepo + + return nil +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go deleted file mode 100644 index 4cafc51..0000000 --- a/internal/cmd/root.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright © 2025 Ahmad Anvari -*/ -package cmd - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var cfgFile string - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "futureq", - Short: "FutureQ server", - Long: `This project involves building a scheduled message triggering service that integrates with message queues like RabbitMQ or Kafka. The service consumes messages from the queue, schedules them for future delivery based on a specified trigger time, and ensures they are dispatched at the correct moment. This is particularly useful for applications requiring delayed message delivery, time-based event triggering, or task scheduling.`, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - err := rootCmd.Execute() - if err != nil { - os.Exit(1) - } -} - -func init() { - cobra.OnInitialize(initConfig) - - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.futureq.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} - -// initConfig reads in config file and ENV variables if set. -func initConfig() { - if cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(cfgFile) - } else { - // Find home directory. - home, err := os.UserHomeDir() - cobra.CheckErr(err) - - // Search config in home directory with name ".futureq" (without extension). - viper.AddConfigPath(home) - viper.SetConfigType("yaml") - viper.SetConfigName(".futureq") - } - - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) - } -} diff --git a/internal/cmd/start.go b/internal/cmd/start.go deleted file mode 100644 index ef11719..0000000 --- a/internal/cmd/start.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright © 2025 NAME HERE -*/ -package cmd - -import ( - "github.com/spf13/cobra" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - - "github.com/futureq-io/futureq/internal/config" - "github.com/futureq-io/futureq/internal/q" - "github.com/futureq-io/futureq/internal/storage" - "github.com/futureq-io/futureq/internal/ticker" -) - -// startCmd represents the server command -var startCmd = &cobra.Command{ - Use: "start", - Short: "Start the server", - Run: startRun, -} - -var ( - configFile *string -) - -func init() { - configFile = startCmd.Flags().StringP("config", "c", "", "Path to config file") - - rootCmd.AddCommand(startCmd) -} - -func startRun(_ *cobra.Command, _ []string) { - var logger *zap.Logger - - loggerConfig := zap.NewProductionConfig() - loggerConfig.DisableCaller = true - loggerConfig.DisableStacktrace = true - loggerConfig.Level = zap.NewAtomicLevelAt(zap.InfoLevel) - loggerConfig.EncoderConfig.TimeKey = "time" - loggerConfig.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder - logger, _ = loggerConfig.Build() - defer func() { - _ = logger.Sync() - }() - - cfg, err := config.PrepareConfig(configFile) - if err != nil { - logger.Fatal("error loading config", zap.Error(err)) - } - - // Post setup of logger after parsing the config - lvl, err := zap.ParseAtomicLevel(cfg.Observability.Logging.Level) - if err != nil { - logger.With(zap.Error(err)).Error("invalid observability.logging.level, continuing with default level: info") - } else { - loggerConfig.Level = lvl - logger, _ = loggerConfig.Build() - } - - taskStorage := storage.NewMemoryArray(cfg.Persistence) - err = taskStorage.InitiatePersistence() - if err != nil { - logger.Fatal("error initializing persistence", zap.Error(err)) - } - - if cfg.RabbitMQ != nil { - rabbitmqQ := q.NewRabbitMQ(*cfg.RabbitMQ, logger.Named("rabbitmq"), taskStorage) - defer rabbitmqQ.Close() - - err := rabbitmqQ.Connect() - if err != nil { - logger.Fatal("error connecting to rabbitmq", zap.Error(err)) - } - - err = rabbitmqQ.Consume() - if err != nil { - logger.Fatal("error consuming rabbitmq", zap.Error(err)) - } - - t := ticker.NewTicker(taskStorage, rabbitmqQ) - - go t.Tick() - } - - logger.Info("starting server") - var forever chan struct{} - - <-forever -} diff --git a/internal/config/config.go b/internal/config/config.go index 54c121e..6e01dd2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,19 +3,111 @@ package config import ( "bytes" "fmt" + "os" + "strconv" "strings" + "time" "github.com/spf13/viper" - "gopkg.in/yaml.v3" + "gopkg.in/yaml.v2" +) + +type AckLevel = string + +const ( + Quorum AckLevel = "Quorum" + NoAck AckLevel = "NoAck" ) type Config struct { + Server Server `mapstructure:"server" yaml:"server"` Observability Observability `mapstructure:"observability" yaml:"observability"` - Persistence Persistence `mapstructure:"persistence" yaml:"persistence"` - RabbitMQ *RabbitMQ `mapstructure:"rabbitmq" yaml:"rabbitmq"` + Storage Storage `mapstructure:"storage" yaml:"storage"` + Raft Raft `mapstructure:"raft" yaml:"raft"` + Consumer Consumer `mapstructure:"consumer" yaml:"consumer"` +} + +type Server struct { + Listen string `mapstructure:"listen" yaml:"listen"` + MaxConns uint32 `mapstructure:"maxConns" yaml:"maxConns"` + Timeout time.Duration `mapstructure:"timeout" yaml:"timeout"` + MaxRecvSizeKB int `mapstructure:"maxRecvSizeKb" yaml:"maxRecvSizeKb"` + MaxSendSizeKB int `mapstructure:"maxSendSizeKb" yaml:"maxSendSizeKb"` +} + +type Observability struct { + Logger Logger `mapstructure:"logger" yaml:"logger"` + Metrics Metrics `mapstructure:"metrics" yaml:"metrics"` +} + +type Metrics struct { + Addr string `mapstructure:"addr" yaml:"addr"` +} + +type Logger struct { + Level string `mapstructure:"level" yaml:"level"` +} + +type Storage struct { + MinAckLevel AckLevel `mapstructure:"minAckLevel" yaml:"minAckLevel"` + Persist bool `mapstructure:"persist" yaml:"persist"` + TimeBucketSize time.Duration `mapstructure:"timeBucketSize" yaml:"timeBucketSize"` + Type string `mapstructure:"type" yaml:"type"` + Pebble Pebble `mapstructure:"pebble" yaml:"pebble"` + Bolt Bolt +} + +type Bolt struct { + DataPath string `mapstructure:"dataPath" yaml:"dataPath"` + DefaultBucket string `mapstructure:"defaultBucket" yaml:"defaultBucket"` +} + +type Pebble struct { + DisableWAL bool `mapstructure:"disableWAL" yaml:"disableWAL"` + DataPath string `mapstructure:"dataPath" yaml:"dataPath"` + CacheSizeMB int64 `mapstructure:"cacheSizeMb" yaml:"cacheSizeMb"` + InMemTableSizeMB uint64 `mapstructure:"inMemoryTableSizeMb" yaml:"inMemoryTableSizeMb"` +} + +type Raft struct { + Enabled bool `mapstructure:"enabled" yaml:"enabled"` + NodeID uint64 `mapstructure:"nodeId" yaml:"nodeId"` + ClusterID uint64 `mapstructure:"clusterId" yaml:"clusterId"` + ListenAddress string `mapstructure:"listenAddress" yaml:"listenAddress"` + DataPath string `mapstructure:"dataPath" yaml:"dataPath"` + InitialMembers map[uint64]string `mapstructure:"initialMembers" yaml:"initialMembers"` + + // RTTMillisecond is the average round-trip latency between Raft peers in + // milliseconds. Dragonboat uses this to calibrate election timeouts and + // heartbeat intervals. Lower values mean faster leader failover but + // higher network overhead. Default: 200. + RTTMillisecond uint64 `mapstructure:"rttMillisecond" yaml:"rttMillisecond"` + + SnapshotEntries uint64 `mapstructure:"snapShotEntries" yaml:"snapShotEntries"` + CompactionOverhead uint64 `mapstructure:"compactionOverHead" yaml:"compactionOverHead"` +} + +// Consumer holds configuration for the message dispatch subsystem. +type Consumer struct { + // DispatchPollIntervalMs is how long the dispatcher sleeps between scan + // passes when no ready messages were found. Shorter values reduce delivery + // latency at the cost of more Pebble iterator overhead. Default: 50ms. + DispatchPollIntervalMs uint64 `mapstructure:"dispatchPollIntervalMs" yaml:"dispatchPollIntervalMs"` + + // DeleteBatchIntervalMs is how often the batched deleter flushes accumulated + // acknowledged-message keys via Raft. Default: 500ms. + DeleteBatchIntervalMs uint64 `mapstructure:"deleteBatchIntervalMs" yaml:"deleteBatchIntervalMs"` + + // InFlightTimeoutMs is the duration after which a dispatched-but-unacknowledged + // message is considered abandoned and eligible for re-dispatch. Default: 5000ms. + InFlightTimeoutMs uint64 `mapstructure:"inFlightTimeoutMs" yaml:"inFlightTimeoutMs"` + + // TTLJanitorIntervalMs is how often the TTL janitor performs a full Pebble + // scan to remove expired messages that were never consumed. Default: 60000ms. + TTLJanitorIntervalMs uint64 `mapstructure:"ttlJanitorIntervalMs" yaml:"ttlJanitorIntervalMs"` } -func PrepareConfig(path *string) (*Config, error) { +func Load(path string) (*Config, error) { var c Config v := viper.New() @@ -25,7 +117,7 @@ func PrepareConfig(path *string) (*Config, error) { v.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) v.AutomaticEnv() - defaultConfigBytes, err := yaml.Marshal(&defaultConfig) + defaultConfigBytes, err := yaml.Marshal(defaultConfig) if err != nil { return nil, fmt.Errorf("error unmarshalling default config: %w", err) } @@ -35,8 +127,8 @@ func PrepareConfig(path *string) (*Config, error) { return nil, fmt.Errorf("error reading default config: %w", err) } - if path != nil && *path != "" { - v.SetConfigFile(*path) + if path != "" { + v.SetConfigFile(path) err = v.MergeInConfig() if err != nil { return nil, fmt.Errorf("error merge config: %w", err) @@ -48,5 +140,84 @@ func PrepareConfig(path *string) (*Config, error) { return nil, fmt.Errorf("error unmarshalling config: %w", err) } + // Explicit env overrides — Viper's AutomaticEnv+Unmarshal does not reliably + // override map values that are already present in the config file, so we + // handle the ones we care about explicitly here. + if nodeID := os.Getenv("FUTUREQ_RAFT_NODEID"); nodeID != "" { + if v, err := strconv.ParseUint(nodeID, 10, 64); err == nil { + c.Raft.NodeID = v + } + } + if raftAddr := os.Getenv("FUTUREQ_RAFT_LISTENADDRESS"); raftAddr != "" { + c.Raft.ListenAddress = raftAddr + } + + if err := c.runPostLoadHooks(); err != nil { + return nil, fmt.Errorf("failed to run post load hooks for config: %w", err) + } + + if err := c.validate(); err != nil { + return nil, fmt.Errorf("error validating config: %w", err) + } + return &c, nil } + +func (c *Config) validate() error { + if err := c.validateStorage(); err != nil { + return err + } + + if err := c.validateRaft(); err != nil { + return err + } + + return nil +} + +func (c *Config) validateRaft() error { + if c.Raft.SnapshotEntries == 0 { + return fmt.Errorf("raft's snapshot entries cannot be zero") + } + + if c.Raft.CompactionOverhead == 0 { + return fmt.Errorf("raft's compaction overhead cannot be zero") + } + + if c.Raft.RTTMillisecond == 0 { + return fmt.Errorf("raft's rtt millisecond cannot be zero") + } + + return nil +} + +func (c *Config) validateStorage() error { + if c.Storage.Persist && c.Storage.Pebble.DataPath == "" { + return fmt.Errorf("pebble's data path cannot be empty when persist is true") + } + + if c.Storage.TimeBucketSize < 1*time.Millisecond { + c.Storage.TimeBucketSize = 0 + } + + if !c.Raft.Enabled { + if c.Storage.Pebble.DisableWAL { + return fmt.Errorf("WAL can not be disabled on single node setup without raft") + } + } + + if c.Storage.Type != "pebble" && c.Storage.Type != "bolt" { + return fmt.Errorf("storage type can only be in (pebble, bolt)") + } + + return nil +} + +func (c *Config) runPostLoadHooks() error { + if !c.Storage.Persist { + c.Storage.Pebble.DataPath = "" + c.Storage.Bolt.DataPath = "" + } + + return nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..2b0dc87 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,39 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/suite" + "gopkg.in/yaml.v2" +) + +type ConfigSuite struct { + suite.Suite +} + +func TestConfigSuite(t *testing.T) { + suite.Run(t, new(ConfigSuite)) +} + +func (s *ConfigSuite) TestExampleConfigMatchesDefault() { + require := s.Require() + + examplePath := filepath.Join("../..", "config.example.yaml") + exampleBytes, err := os.ReadFile(examplePath) + require.NoError(err) + + var exampleMap map[string]interface{} + err = yaml.Unmarshal(exampleBytes, &exampleMap) + require.NoError(err) + + defaultBytes, err := yaml.Marshal(defaultConfig) + require.NoError(err) + + var defaultMap map[string]interface{} + err = yaml.Unmarshal(defaultBytes, &defaultMap) + require.NoError(err) + + require.Equal(defaultMap, exampleMap) +} diff --git a/internal/config/default.go b/internal/config/default.go index fff697a..607cce9 100644 --- a/internal/config/default.go +++ b/internal/config/default.go @@ -1,13 +1,58 @@ package config +import "time" + var defaultConfig = Config{ + Server: Server{ + Listen: "0.0.0.0:8443", + MaxConns: 10, + Timeout: 5 * time.Second, + MaxSendSizeKB: 100, + MaxRecvSizeKB: 100, + }, + Observability: Observability{ - Logging: Logging{ + Logger: Logger{ Level: "info", }, + Metrics: Metrics{ + Addr: "0.0.0.0:9090", + }, }, - Persistence: Persistence{ - Path: "./data", + + Storage: Storage{ + MinAckLevel: Quorum, + Persist: true, + TimeBucketSize: 1 * time.Millisecond, + Type: "pebble", + Pebble: Pebble{ + DisableWAL: false, + DataPath: "./data", + CacheSizeMB: 16, + InMemTableSizeMB: 64, + }, + Bolt: Bolt{ + DataPath: "./data", + DefaultBucket: "futureq", + }, + }, + + Raft: Raft{ + Enabled: false, + NodeID: 1, + ClusterID: 1, + ListenAddress: "0.0.0.0:50005", + DataPath: "./raft-data", + InitialMembers: map[uint64]string{1: "0.0.0.0:50005"}, + RTTMillisecond: 200, + SnapshotEntries: 10000, + CompactionOverhead: 5000, + }, + + Consumer: Consumer{ + DispatchPollIntervalMs: 50, + DeleteBatchIntervalMs: 500, + InFlightTimeoutMs: 5000, + TTLJanitorIntervalMs: 60000, }, - RabbitMQ: nil, } diff --git a/internal/config/observability.go b/internal/config/observability.go deleted file mode 100644 index b08d29a..0000000 --- a/internal/config/observability.go +++ /dev/null @@ -1,9 +0,0 @@ -package config - -type Observability struct { - Logging Logging `mapstructure:"logging" yaml:"logging"` -} - -type Logging struct { - Level string `mapstructure:"level" yaml:"level"` -} diff --git a/internal/config/persistence.go b/internal/config/persistence.go deleted file mode 100644 index 547dcba..0000000 --- a/internal/config/persistence.go +++ /dev/null @@ -1,5 +0,0 @@ -package config - -type Persistence struct { - Path string `mapstructure:"path" yaml:"path"` -} diff --git a/internal/config/rabbitmq.go b/internal/config/rabbitmq.go deleted file mode 100644 index ec5f5b0..0000000 --- a/internal/config/rabbitmq.go +++ /dev/null @@ -1,28 +0,0 @@ -package config - -import ( - "fmt" -) - -type RabbitMQ struct { - RabbitMQServer RabbitMQServer `mapstructure:"rabbitmq_server" yaml:"rabbitmq_server"` - RabbitMQDataExchange RabbitMQDataExchange `mapstructure:"rabbitmq_data_exchange" yaml:"rabbitmq_data_exchange"` -} - -type RabbitMQServer struct { - Host string `mapstructure:"host" yaml:"host"` - Port uint `mapstructure:"port" yaml:"port"` - Username string `mapstructure:"username" yaml:"username"` - Password string `mapstructure:"password" yaml:"password"` - VirtualHost string `mapstructure:"virtual_host" yaml:"virtual_host"` -} - -type RabbitMQDataExchange struct { - ConsumeQueueName string `mapstructure:"consume_queue_name" yaml:"consume_queue_name"` - DeclareQueue bool `mapstructure:"declare_queue" yaml:"declare_queue"` - ProduceQueueName string `mapstructure:"produce_queue_name" yaml:"produce_queue_name"` -} - -func (r RabbitMQServer) ConnectionURI() string { - return fmt.Sprintf("amqp://%s:%s@%s:%d/%s", r.Username, r.Password, r.Host, r.Port, r.VirtualHost) -} diff --git a/internal/dispatcher/deleter.go b/internal/dispatcher/deleter.go new file mode 100644 index 0000000..d8852a5 --- /dev/null +++ b/internal/dispatcher/deleter.go @@ -0,0 +1,166 @@ +package dispatcher + +import ( + "context" + "sync" + "time" + + "github.com/futureq-io/futureq/internal/metrics" + "github.com/futureq-io/futureq/internal/raft/event" + "github.com/futureq-io/futureq/internal/storage" + "go.uber.org/zap" +) + +// DeleteBackend abstracts how deletions are persisted. +// In Raft mode, deletions are proposed to the cluster. In standalone mode, +// they are written directly to the local storage engine. +type DeleteBackend interface { + // DeleteKeys atomically removes the given keys from storage. + // Returns nil on success, or an error if the deletion could not be + // completed. On error, the keys are NOT removed and should be retried. + DeleteKeys(keys [][]byte) error +} + +// raftDeleteBackend routes deletions through Raft consensus. +type raftDeleteBackend struct { + propose func(cmd []byte) error + logger *zap.Logger +} + +// NewRaftDeleteBackend returns a DeleteBackend that replicates deletions +// via Raft DeleteBatchCmd. +func NewRaftDeleteBackend(propose func(cmd []byte) error, logger *zap.Logger) DeleteBackend { + return &raftDeleteBackend{ + propose: propose, + logger: logger.Named("raft_delete"), + } +} + +func (b *raftDeleteBackend) DeleteKeys(keys [][]byte) error { + cmd, err := raft.MarshalDeleteBatchCmd(keys) + if err != nil { + return err + } + return b.propose(cmd) +} + +// directDeleteBackend writes deletions directly to the local storage engine. +type directDeleteBackend struct { + db storage.DB + logger *zap.Logger +} + +// NewDirectDeleteBackend returns a DeleteBackend that deletes keys directly +// from the local DB (single-node mode). +func NewDirectDeleteBackend(db storage.DB, logger *zap.Logger) DeleteBackend { + return &directDeleteBackend{ + db: db, + logger: logger.Named("direct_delete"), + } +} + +func (b *directDeleteBackend) DeleteKeys(keys [][]byte) error { + batch := b.db.NewBatch() + defer batch.Close() //nolint:errcheck + + for _, key := range keys { + if err := batch.Delete(key); err != nil { + b.logger.Error("failed to mark key for deletion", zap.Error(err)) + } + } + + return batch.Commit(storage.Sync) +} + +// ─── Deleter ───────────────────────────────────────────────────────────────── + +// Deleter accumulates acknowledged-message keys and periodically flushes them +// as a single batch through the configured DeleteBackend. +// +// Routing deletions through Raft ensures that all replicas remove acknowledged +// messages atomically, preventing a new leader from re-dispatching a message +// that was already acknowledged before a failover. +type Deleter struct { + backend DeleteBackend + logger *zap.Logger + interval time.Duration + + mu sync.Mutex + pending [][]byte + + // OnDelete is called after keys are successfully deleted, with copies of + // each key. Used to remove entries from the dispatcher's in-flight map. + OnDelete func(key []byte) +} + +// NewDeleter constructs a Deleter with the given backend and flush interval. +func NewDeleter(backend DeleteBackend, interval time.Duration, logger *zap.Logger) *Deleter { + return &Deleter{ + backend: backend, + logger: logger.Named("deleter"), + interval: interval, + pending: make([][]byte, 0, 1024), + } +} + +// MarkDeleted enqueues a key for batched deletion. The key is the 24-byte +// storage key received as the delivery_tag from the consumer's AckRequest. +func (d *Deleter) MarkDeleted(key []byte) { + keyCopy := make([]byte, len(key)) + copy(keyCopy, key) + + d.mu.Lock() + d.pending = append(d.pending, keyCopy) + d.mu.Unlock() +} + +// Run starts the batched delete loop. It blocks until ctx is cancelled. +func (d *Deleter) Run(ctx context.Context) { + ticker := time.NewTicker(d.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + d.flush() + return + case <-ticker.C: + d.flush() + } + } +} + +// flush drains the pending queue and deletes the accumulated keys via the +// configured backend. On success, invokes the OnDelete callback for each key. +func (d *Deleter) flush() { + d.mu.Lock() + if len(d.pending) == 0 { + d.mu.Unlock() + return + } + keysToFlush := d.pending + d.pending = make([][]byte, 0, 1024) + d.mu.Unlock() + + if err := d.backend.DeleteKeys(keysToFlush); err != nil { + d.logger.Error("failed to delete batch", + zap.Error(err), + zap.Int("count", len(keysToFlush)), + ) + metrics.DeleteFailuresTotal.Inc() + // Re-enqueue failed keys for retry on next flush. + d.mu.Lock() + d.pending = append(keysToFlush, d.pending...) + d.mu.Unlock() + return + } + + metrics.DeleteBatchSize.Observe(float64(len(keysToFlush))) + d.logger.Debug("flushed delete batch", zap.Int("count", len(keysToFlush))) + + if d.OnDelete != nil { + for _, key := range keysToFlush { + d.OnDelete(key) + } + } +} diff --git a/internal/dispatcher/deleter_test.go b/internal/dispatcher/deleter_test.go new file mode 100644 index 0000000..a84d6a1 --- /dev/null +++ b/internal/dispatcher/deleter_test.go @@ -0,0 +1,245 @@ +package dispatcher + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/storage" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" +) + +type DeleterSuite struct { + suite.Suite +} + +func TestDeleterSuite(t *testing.T) { + suite.Run(t, new(DeleterSuite)) +} + +// ─── DirectDeleteBackend ───────────────────────────────────────────────────── + +func (s *DeleterSuite) TestDirectDeleteBackend_RemovesKeys() { + require := s.Require() + + db, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + require.NoError(err) + defer db.Close() + + // Seed a key. + b := db.NewBatch() + require.NoError(b.Set([]byte("to-delete"), []byte("v"))) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + before, closer, err := db.Get([]byte("to-delete")) + require.NoError(err) + require.Equal([]byte("v"), before) + closer.Close() + + backend := NewDirectDeleteBackend(db, zap.NewNop()) + require.NoError(backend.DeleteKeys([][]byte{[]byte("to-delete")})) + + _, _, err = db.Get([]byte("to-delete")) + require.Error(err, "expected key to be deleted") +} + +func (s *DeleterSuite) TestDirectDeleteBackend_MultipleKeys() { + require := s.Require() + + db, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + require.NoError(err) + defer db.Close() + + b := db.NewBatch() + for _, k := range []string{"k1", "k2", "k3"} { + require.NoError(b.Set([]byte(k), []byte("v"))) + } + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + backend := NewDirectDeleteBackend(db, zap.NewNop()) + require.NoError(backend.DeleteKeys([][]byte{ + []byte("k1"), []byte("k2"), []byte("k3"), + })) + + for _, k := range []string{"k1", "k2", "k3"} { + _, _, err := db.Get([]byte(k)) + require.Error(err) + } +} + +func (s *DeleterSuite) TestDirectDeleteBackend_EmptyList_Succeeds() { + require := s.Require() + + db, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + require.NoError(err) + defer db.Close() + + backend := NewDirectDeleteBackend(db, zap.NewNop()) + require.NoError(backend.DeleteKeys([][]byte{})) +} + +// ─── RaftDeleteBackend ────────────────────────────────────────────────────── + +func (s *DeleterSuite) TestRaftDeleteBackend_ProposalsMarshalledCommand() { + require := s.Require() + + var captured []byte + proposeFn := func(cmd []byte) error { + captured = append([]byte(nil), cmd...) + return nil + } + + backend := NewRaftDeleteBackend(proposeFn, zap.NewNop()) + + key := make([]byte, 24) // required key length for MarshalDeleteBatchCmd + require.NoError(backend.DeleteKeys([][]byte{key})) + + require.NotNil(captured, "propose must be called") + require.Equal(byte(1), captured[0], "first byte should be DeleteBatchCmd type (1)") +} + +func (s *DeleterSuite) TestRaftDeleteBackend_ProposalsErrorPropagates() { + require := s.Require() + + sentinel := errors.New("propose failed") + proposeFn := func(cmd []byte) error { return sentinel } + + backend := NewRaftDeleteBackend(proposeFn, zap.NewNop()) + key := make([]byte, 24) + + err := backend.DeleteKeys([][]byte{key}) + require.Error(err) + require.Contains(err.Error(), sentinel.Error()) +} + +func (s *DeleterSuite) TestRaftDeleteBackend_InvalidKeyLength_Fails() { + require := s.Require() + + proposeFn := func(cmd []byte) error { return nil } + backend := NewRaftDeleteBackend(proposeFn, zap.NewNop()) + + err := backend.DeleteKeys([][]byte{[]byte("short-key")}) + require.Error(err, "a non-24-byte key must be rejected") +} + +// ─── Deleter ───────────────────────────────────────────────────────────────── + +// mockBackend records every DeleteKeys call and can be set to fail. +type mockBackend struct { + calls atomic.Int32 + err error + keys [][]byte +} + +func (m *mockBackend) DeleteKeys(keys [][]byte) error { + m.calls.Add(1) + m.keys = keys + return m.err +} + +func (s *DeleterSuite) TestDeleter_MarkDeleted_AccumulatesKeys() { + require := s.Require() + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + + d.MarkDeleted([]byte("a")) + d.MarkDeleted([]byte("b")) + + d.mu.Lock() + defer d.mu.Unlock() + require.Len(d.pending, 2) +} + +func (s *DeleterSuite) TestDeleter_Run_FlushesOnInterval() { + require := s.Require() + + backend := &mockBackend{} + d := NewDeleter(backend, 10*time.Millisecond, zap.NewNop()) + + d.MarkDeleted([]byte("x")) + d.MarkDeleted([]byte("y")) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan struct{}) + go func() { d.Run(ctx); close(done) }() + + // Wait for at least one flush. + require.Eventually(func() bool { + return backend.calls.Load() >= 1 + }, time.Second, 5*time.Millisecond) + + cancel() + <-done +} + +func (s *DeleterSuite) TestDeleter_RetriesFailedFlush() { + require := s.Require() + + backend := &mockBackend{err: errors.New("boom")} + d := NewDeleter(backend, time.Hour, zap.NewNop()) // long interval — only ctx cancel triggers flush + + d.MarkDeleted([]byte("retry-me")) + + // Trigger a flush manually via Run + Cancel. + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { d.Run(ctx); close(done) }() + cancel() + <-done + + // After a failed flush the key must be re-enqueued for retry. + d.mu.Lock() + defer d.mu.Unlock() + require.Len(d.pending, 1, "failed keys must be re-enqueued") + require.Equal([]byte("retry-me"), d.pending[0]) +} + +func (s *DeleterSuite) TestDeleter_OnDelete_CalledForEachKey() { + require := s.Require() + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + + var deleted []string + d.OnDelete = func(key []byte) { deleted = append(deleted, string(key)) } + + d.MarkDeleted([]byte("k1")) + d.MarkDeleted([]byte("k2")) + + // Trigger flush via ctx cancel. + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { d.Run(ctx); close(done) }() + cancel() + <-done + + require.Len(deleted, 2) + require.Contains(deleted, "k1") + require.Contains(deleted, "k2") +} + +func (s *DeleterSuite) TestDeleter_MarkDeleted_CopiesKey() { + require := s.Require() + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + + original := []byte("original-key") + d.MarkDeleted(original) + + // Mutating the original slice must not affect the pending copy. + original[0] = 'X' + + d.mu.Lock() + defer d.mu.Unlock() + require.Equal([]byte("original-key"), d.pending[0]) +} diff --git a/internal/dispatcher/dispatcher.go b/internal/dispatcher/dispatcher.go new file mode 100644 index 0000000..e0dad9f --- /dev/null +++ b/internal/dispatcher/dispatcher.go @@ -0,0 +1,293 @@ +package dispatcher + +import ( + "context" + "sync" + "time" + + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + + "github.com/futureq-io/futureq/internal/app" + "github.com/futureq-io/futureq/internal/metrics" + "github.com/futureq-io/futureq/internal/storage" + + "github.com/futureq-io/futureq/pkg/utils" + pb "github.com/futureq-io/protocol/proto/go" + storagepb "github.com/futureq-io/protocol/proto/go/storage" +) + +// inFlightEntry tracks a single message that has been sent to a consumer but +// not yet acknowledged. +type inFlightEntry struct { + dispatchedAt time.Time + topic string +} + +// Dispatcher scans the storage engine for messages that are due for delivery +// and dispatches them to connected consumers via the Hub. +// +// Key design choices: +// - Per-topic range scans using the topic-first key layout +// - Only scans topics with connected consumers (active-topic set from Hub) +// - Tracks in-flight messages per key; cleans up on consumer disconnect +// - Performs TTL checks at dispatch time; expired messages are batched for deletion +// - Snapshot-based iteration (never blocks concurrent writes) +type Dispatcher struct { + db storage.DB + hub *Hub + deleter *Deleter + logger *zap.Logger + interval time.Duration + inFlightTimeout time.Duration + wakeCh chan struct{} + inFlight sync.Map // key: string(pebbleKey) → *inFlightEntry +} + +// NewDispatcher constructs a Dispatcher. +func NewDispatcher( + db storage.DB, + hub *Hub, + deleter *Deleter, + interval time.Duration, + inFlightTimeout time.Duration, + wakeCh chan struct{}, + logger *zap.Logger, +) *Dispatcher { + return &Dispatcher{ + db: db, + hub: hub, + deleter: deleter, + logger: logger.Named("dispatcher"), + interval: interval, + inFlightTimeout: inFlightTimeout, + wakeCh: wakeCh, + } +} + +// RemoveInFlight removes a message from the in-flight tracker by key, making +// it eligible for re-dispatch if it still exists in storage. +func (d *Dispatcher) RemoveInFlight(key []byte) { + d.inFlight.Delete(string(key)) +} + +// RemoveInFlightBatch removes multiple keys from the in-flight tracker. +// Called by the state machine's OnDeleteKeys callback after Raft applies a +// DeleteBatchCmd — at that point the keys are gone from all replicas. +func (d *Dispatcher) RemoveInFlightBatch(keys [][]byte) { + for _, k := range keys { + d.inFlight.Delete(string(k)) + } +} + +// Run is the dispatcher event loop. It blocks until ctx is cancelled. +func (d *Dispatcher) Run(ctx context.Context) { + timer := time.NewTimer(d.interval) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-d.wakeCh: + // A consumer connected — scan immediately. + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + d.dispatchAll() + timer.Reset(d.interval) + case <-timer.C: + dispatched := d.dispatchAll() + if dispatched > 0 { + // More messages may be ready — re-scan without delay. + timer.Reset(0) + } else { + timer.Reset(d.interval) + } + } + } +} + +// dispatchAll performs one full dispatch pass across all active topics. +// Returns the total number of messages dispatched. +func (d *Dispatcher) dispatchAll() int { + if !d.hub.HasConsumers() { + return 0 + } + + // In Raft mode, only the leader dispatches messages. + if !d.isLeader() { + return 0 + } + + activeTopics := d.hub.ActiveTopics() + if len(activeTopics) == 0 { + return 0 + } + + start := time.Now() + nowMs := start.UnixMilli() + totalDispatched := 0 + + for _, topic := range activeTopics { + dispatched := d.dispatchTopic(topic, nowMs) + totalDispatched += dispatched + } + + metrics.DispatchPassDurationMs.Observe(float64(time.Since(start).Milliseconds())) + + return totalDispatched +} + +// isLeader returns true if this node should dispatch messages. +// In standalone mode (no Raft), always returns true. +func (d *Dispatcher) isLeader() bool { + if app.A.NodeHost == nil { + return true + } + shardID := app.A.Config().Raft.ClusterID + leaderID, _, valid, err := app.A.NodeHost.GetLeaderID(shardID) + if err != nil || !valid { + return false + } + return leaderID == app.A.Config().Raft.NodeID +} + +// dispatchTopic scans a single topic's key range and dispatches due messages. +// Returns the number of messages dispatched for this topic. +// +// The scan is bounded to buckets ≤ nowBucket so future (not-yet-due) messages +// are never iterated. The key layout is [topicHash][bucket][eventID], so the +// exclusive upper bound [topicHash][nowBucket+1][0...] covers all and only +// the due messages for this topic. +func (d *Dispatcher) dispatchTopic(topic string, nowMs int64) int { + topicHash := utils.TopicHash(topic) + nowBucket := utils.CalculateBucket(nowMs, app.A.Config().Storage.TimeBucketSize) + + dispatched := 0 + var expiredKeys [][]byte + + // Per-topic range scan over due buckets only: [topicHash, topicHash | nowBucket+1). + // Scan manages the snapshot/iterator lifecycle internally — lower overhead + // than NewIter since there's no manual resource management. + err := d.db.Scan(&storage.IterOptions{ + LowerBound: utils.TopicLowerBound(topicHash), + UpperBound: utils.DueUpperBound(topicHash, nowBucket), + }, func(key, val []byte) error { + // Check in-flight status. + if d.isInFlight(key) { + return nil + } + + // Deserialize the stored message. + var msg storagepb.StoredMessage + if err := proto.Unmarshal(val, &msg); err != nil { + d.logger.Error("failed to unmarshal stored message", zap.Error(err)) + return nil + } + + // TTL check: skip and collect for deletion if expired. + if d.isExpired(&msg, nowMs) { + keyCopy := make([]byte, len(key)) + copy(keyCopy, key) + expiredKeys = append(expiredKeys, keyCopy) + metrics.MessagesExpiredTotal.WithLabelValues(topic, "dispatcher").Inc() + return nil + } + + // Build the queue message. Must copy key — it's only valid during yield. + keyCopy := make([]byte, len(key)) + copy(keyCopy, key) + + qMsg := &pb.QueueMessage{ + Topic: msg.Topic, + Payload: msg.Payload, + DeliveryTag: keyCopy, + EnqueuedAtUnixMs: msg.EnqueuedAtUnixMs, + DelayMs: msg.DelayMs, + } + + // Dispatch to all eligible consumers on this topic. + sentTo := d.hub.DispatchToTopic(topic, qMsg, keyCopy) + if len(sentTo) > 0 { + // Track in-flight for timeout-based redelivery. + d.inFlight.Store(string(keyCopy), &inFlightEntry{ + dispatchedAt: time.Now(), + topic: topic, + }) + dispatched++ + + // Delivery latency: total time from enqueue to dispatch. + latencyMs := float64(nowMs - msg.EnqueuedAtUnixMs) + if latencyMs < 0 { + latencyMs = 0 // clock skew guard + } + metrics.DeliveryLatencyMs.WithLabelValues(topic).Observe(latencyMs) + + // Overhead: how late we are relative to the scheduled time. + scheduledMs := msg.EnqueuedAtUnixMs + msg.DelayMs + overheadMs := float64(nowMs - scheduledMs) + if overheadMs < 0 { + overheadMs = 0 // dispatched early (shouldn't happen, but guard) + } + metrics.DeliveryOverheadMs.WithLabelValues(topic).Observe(overheadMs) + + for _, gid := range sentTo { + metrics.MessagesDispatchedTotal.WithLabelValues(topic, gid).Inc() + metrics.MessagesInFlight.WithLabelValues(topic, gid).Inc() + } + } + + return nil + }) + + if err != nil { + d.logger.Error("scan error", + zap.String("topic", topic), + zap.Error(err), + ) + } + + // Batch-delete expired messages. + if len(expiredKeys) > 0 { + for _, k := range expiredKeys { + d.deleter.MarkDeleted(k) + } + d.logger.Debug("queued expired messages for deletion", + zap.String("topic", topic), + zap.Int("count", len(expiredKeys)), + ) + } + + return dispatched +} + +// isInFlight checks if a key is currently in-flight and not yet timed out. +// If the entry has timed out, it is removed and the key is eligible for +// re-dispatch. +func (d *Dispatcher) isInFlight(key []byte) bool { + entry, exists := d.inFlight.Load(string(key)) + if !exists { + return false + } + + e := entry.(*inFlightEntry) + if time.Since(e.dispatchedAt) < d.inFlightTimeout { + return true + } + + // Timed out — allow re-dispatch. + d.inFlight.Delete(string(key)) + return false +} + +// isExpired returns true if the message's TTL has elapsed. +func (d *Dispatcher) isExpired(msg *storagepb.StoredMessage, nowMs int64) bool { + if msg.TtlMs <= 0 { + return false + } + return nowMs >= msg.EnqueuedAtUnixMs+msg.TtlMs +} diff --git a/internal/dispatcher/dispatcher_test.go b/internal/dispatcher/dispatcher_test.go new file mode 100644 index 0000000..feb54f0 --- /dev/null +++ b/internal/dispatcher/dispatcher_test.go @@ -0,0 +1,266 @@ +package dispatcher + +import ( + "context" + "testing" + "time" + + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/storage" + "github.com/futureq-io/futureq/pkg/utils" + storagepb "github.com/futureq-io/protocol/proto/go/storage" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" +) + +type DispatcherSuite struct { + suite.Suite + db storage.DB +} + +func TestDispatcherSuite(t *testing.T) { + suite.Run(t, new(DispatcherSuite)) +} + +func (s *DispatcherSuite) SetupTest() { + db, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + s.Require().NoError(err) + s.db = db +} + +func (s *DispatcherSuite) TearDownTest() { + if s.db != nil { + s.db.Close() + } +} + +// storeDueMessage writes a StoredMessage that is already due for delivery. +// Returns the generated event key. +func (s *DispatcherSuite) storeDueMessage(topic string, payload []byte, delayMs, ttlMs int64) []byte { + bucketSize := 1 * time.Second + nowMs := time.Now().UnixMilli() + + enqueuedAt := nowMs - 10_000 // enqueued 10s ago + + msg := &storagepb.StoredMessage{ + Topic: topic, + Payload: payload, + EnqueuedAtUnixMs: enqueuedAt, + DelayMs: delayMs, + TtlMs: ttlMs, + } + data, err := proto.Marshal(msg) + s.Require().NoError(err) + + fireAtMs := enqueuedAt + delayMs + bucket := utils.CalculateBucket(fireAtMs, bucketSize) + key := utils.EventKey(bucket, utils.TopicHash(topic), 1) + + b := s.db.NewBatch() + s.Require().NoError(b.Set(key, data)) + s.Require().NoError(b.Commit(storage.Sync)) + s.Require().NoError(b.Close()) + + return key +} + +// ─── isInFlight / RemoveInFlight ───────────────────────────────────────────── + +func (s *DispatcherSuite) TestIsInFlight_NewKey_NotInFlight() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Second, 5*time.Second, wakeCh, zap.NewNop()) + + require.False(disp.isInFlight([]byte("some-key"))) +} + +func (s *DispatcherSuite) TestRemoveInFlight_EvictsEntry() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Second, 5*time.Second, wakeCh, zap.NewNop()) + + key := []byte("my-key") + disp.inFlight.Store(string(key), &inFlightEntry{dispatchedAt: time.Now(), topic: "t"}) + require.True(disp.isInFlight(key)) + + disp.RemoveInFlight(key) + require.False(disp.isInFlight(key)) +} + +func (s *DispatcherSuite) TestRemoveInFlightBatch_RemovesAllKeys() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Second, 5*time.Second, wakeCh, zap.NewNop()) + + keys := [][]byte{[]byte("k1"), []byte("k2"), []byte("k3")} + for _, k := range keys { + disp.inFlight.Store(string(k), &inFlightEntry{dispatchedAt: time.Now(), topic: "t"}) + } + + disp.RemoveInFlightBatch(keys) + + for _, k := range keys { + require.False(disp.isInFlight(k)) + } +} + +func (s *DispatcherSuite) TestIsInFlight_TimedOutEntry_ReturnsFalse() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + // Very short in-flight timeout. + disp := NewDispatcher(s.db, hub, d, time.Second, 1*time.Millisecond, wakeCh, zap.NewNop()) + + key := []byte("old-key") + // Entry dispatched in the past → has timed out. + disp.inFlight.Store(string(key), &inFlightEntry{ + dispatchedAt: time.Now().Add(-time.Hour), + topic: "t", + }) + + require.False(disp.isInFlight(key), "timed-out entry must not be considered in-flight") + + // Entry must also be evicted from the map. + _, exists := disp.inFlight.Load(string(key)) + require.False(exists) +} + +// ─── isExpired ─────────────────────────────────────────────────────────────── + +func (s *DispatcherSuite) TestIsExpired_ZeroTTL_NeverExpires() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Second, 5*time.Second, wakeCh, zap.NewNop()) + + msg := &storagepb.StoredMessage{ + EnqueuedAtUnixMs: 1, + TtlMs: 0, + } + require.False(disp.isExpired(msg, time.Now().UnixMilli())) +} + +func (s *DispatcherSuite) TestIsExpired_TTLElapsed_True() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Second, 5*time.Second, wakeCh, zap.NewNop()) + + nowMs := time.Now().UnixMilli() + msg := &storagepb.StoredMessage{ + EnqueuedAtUnixMs: nowMs - 10_000, + TtlMs: 5_000, + } + require.True(disp.isExpired(msg, nowMs)) +} + +func (s *DispatcherSuite) TestIsExpired_TTLNotYetElapsed_False() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Second, 5*time.Second, wakeCh, zap.NewNop()) + + nowMs := time.Now().UnixMilli() + msg := &storagepb.StoredMessage{ + EnqueuedAtUnixMs: nowMs, + TtlMs: 60_000, + } + require.False(disp.isExpired(msg, nowMs)) +} + +// ─── dispatchAll (standalone mode, no raft) ───────────────────────────────── +// +// These tests exercise the standalone path where app.A.NodeHost == nil, so +// isLeader() always returns true. + +func (s *DispatcherSuite) TestDispatchAll_NoConsumers_ReturnsZero() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Second, 5*time.Second, wakeCh, zap.NewNop()) + + require.Equal(0, disp.dispatchAll()) +} + +// ─── Run (event loop) ─────────────────────────────────────────────────────── + +func (s *DispatcherSuite) TestRun_CancelStopsLoop() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + disp := NewDispatcher(s.db, hub, d, time.Hour, 5*time.Second, wakeCh, zap.NewNop()) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { disp.Run(ctx); close(done) }() + + cancel() + select { + case <-done: + // OK + case <-time.After(time.Second): + require.Fail("Run did not exit after ctx cancel") + } +} + +func (s *DispatcherSuite) TestRun_WakeSignalTriggersImmediateScan() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + hub := NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) + backend := NewDirectDeleteBackend(s.db, zap.NewNop()) + d := NewDeleter(backend, time.Hour, zap.NewNop()) + // Long interval — only wakeCh can trigger a scan in reasonable time. + disp := NewDispatcher(s.db, hub, d, time.Hour, 5*time.Second, wakeCh, zap.NewNop()) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { disp.Run(ctx); close(done) }() + + // Register a consumer so HasConsumers returns true. + consumerCh := make(chan interface{}, 1) + _ = consumerCh + // (dispatcher has no messages to dispatch — just verifying Run doesn't deadlock) + wakeCh <- struct{}{} + + // Give it a moment to process the wake signal. + time.Sleep(50 * time.Millisecond) + + cancel() + select { + case <-done: + case <-time.After(time.Second): + require.Fail("Run did not exit after cancel") + } +} diff --git a/internal/dispatcher/hub.go b/internal/dispatcher/hub.go new file mode 100644 index 0000000..4c37375 --- /dev/null +++ b/internal/dispatcher/hub.go @@ -0,0 +1,362 @@ +package dispatcher + +import ( + "bytes" + "fmt" + "sync" + + pb "github.com/futureq-io/protocol/proto/go" + "go.uber.org/zap" +) + +// ─── Dispatch Strategy ─────────────────────────────────────────────────────── + +// DispatchStrategy selects one consumer from a group to receive a message. +// Implementations must be safe for concurrent use. +type DispatchStrategy interface { + // Select returns the consumer that should receive the message, or nil if + // no consumer is available. The candidates slice is a snapshot — safe to + // read without holding locks. + Select(candidates []*ConsumerEntry, msg *pb.QueueMessage) *ConsumerEntry +} + +// RoundRobinStrategy dispatches messages to consumers in rotating order. +type RoundRobinStrategy struct { + mu sync.Mutex + next map[string]uint64 // groupKey → next index +} + +// NewRoundRobinStrategy returns a RoundRobinStrategy. +func NewRoundRobinStrategy() *RoundRobinStrategy { + return &RoundRobinStrategy{ + next: make(map[string]uint64), + } +} + +// Select picks the next consumer in round-robin order for the given group. +func (s *RoundRobinStrategy) Select(candidates []*ConsumerEntry, _ *pb.QueueMessage) *ConsumerEntry { + if len(candidates) == 0 { + return nil + } + + // Use the first candidate's group key for the round-robin counter. + groupKey := candidates[0].GroupKey() + + s.mu.Lock() + idx := s.next[groupKey] + s.next[groupKey] = (idx + 1) % uint64(len(candidates)) + s.mu.Unlock() + + return candidates[idx%uint64(len(candidates))] +} + +// ─── Consumer Entry ────────────────────────────────────────────────────────── + +// ConsumerEntry holds one consumer's state. +type ConsumerEntry struct { + ID string + Topic string + Group string + Ch chan *pb.QueueMessage +} + +// GroupKey returns a canonical key for the consumer's (topic, group) pair. +// Universal consumers (empty group) each get their own unique key so they +// never compete with each other. +func (c *ConsumerEntry) GroupKey() string { + if c.Group == "" { + return fmt.Sprintf("%s|__universal__|%s", c.Topic, c.ID) + } + return fmt.Sprintf("%s|%s", c.Topic, c.Group) +} + +// IsUniversal returns true if this consumer receives every message on the +// topic (no group — fan-out). +func (c *ConsumerEntry) IsUniversal() bool { + return c.Group == "" +} + +// ─── Topic Subscription ───────────────────────────────────────────────────── + +// TopicSubscription tracks all consumers for a single topic, organised by +// group. Universal consumers (empty group) are stored individually. +type TopicSubscription struct { + // groups: groupID → []*ConsumerEntry (competing consumers) + groups map[string][]*ConsumerEntry + + // universal: consumerID → *ConsumerEntry (fan-out consumers) + universal map[string]*ConsumerEntry +} + +// newTopicSubscription returns an empty TopicSubscription. +func newTopicSubscription() *TopicSubscription { + return &TopicSubscription{ + groups: make(map[string][]*ConsumerEntry), + universal: make(map[string]*ConsumerEntry), + } +} + +// add inserts a consumer into the appropriate bucket. +func (ts *TopicSubscription) add(c *ConsumerEntry) { + if c.IsUniversal() { + ts.universal[c.ID] = c + return + } + ts.groups[c.Group] = append(ts.groups[c.Group], c) +} + +// remove deletes a consumer. Returns true if the subscription is now empty. +func (ts *TopicSubscription) remove(c *ConsumerEntry) bool { + if c.IsUniversal() { + delete(ts.universal, c.ID) + } else { + group := ts.groups[c.Group] + for i, ce := range group { + if ce.ID == c.ID { + ts.groups[c.Group] = append(group[:i], group[i+1:]...) + break + } + } + if len(ts.groups[c.Group]) == 0 { + delete(ts.groups, c.Group) + } + } + return ts.isEmpty() +} + +// isEmpty returns true if no consumers remain on this topic. +func (ts *TopicSubscription) isEmpty() bool { + return len(ts.groups) == 0 && len(ts.universal) == 0 +} + +// groupSnapshot returns a snapshot of all groups (non-universal). +func (ts *TopicSubscription) groupSnapshot() map[string][]*ConsumerEntry { + snap := make(map[string][]*ConsumerEntry, len(ts.groups)) + for gid, consumers := range ts.groups { + cp := make([]*ConsumerEntry, len(consumers)) + copy(cp, consumers) + snap[gid] = cp + } + return snap +} + +// universalSnapshot returns a snapshot of all universal consumers. +func (ts *TopicSubscription) universalSnapshot() []*ConsumerEntry { + snap := make([]*ConsumerEntry, 0, len(ts.universal)) + for _, c := range ts.universal { + snap = append(snap, c) + } + return snap +} + +// ─── Hub ───────────────────────────────────────────────────────────────────── + +// Hub manages consumer connections indexed by topic. +// +// Delivery semantics: +// - Universal consumers (empty group): each receives every message (fan-out). +// - Grouped consumers: within each group, exactly one consumer receives each +// message (competing consumers). Different groups each get an independent +// copy (fan-out across groups). +type Hub struct { + mu sync.RWMutex + + // topics: topic → *TopicSubscription + topics map[string]*TopicSubscription + + // byID: consumerID → *ConsumerEntry (fast lookup for unregister) + byID map[string]*ConsumerEntry + + // inFlightByConsumer: consumerID → [][]byte (keys in-flight to that consumer) + inFlightByConsumer map[string][][]byte + inFlightMu sync.Mutex + + strategy DispatchStrategy + logger *zap.Logger + wakeCh chan struct{} +} + +// NewHub constructs a Hub. wakeCh is signalled when a new consumer connects, +// causing the dispatcher to immediately scan for due messages. +func NewHub(strategy DispatchStrategy, logger *zap.Logger, wakeCh chan struct{}) *Hub { + return &Hub{ + topics: make(map[string]*TopicSubscription), + byID: make(map[string]*ConsumerEntry), + inFlightByConsumer: make(map[string][][]byte), + strategy: strategy, + logger: logger.Named("hub"), + wakeCh: wakeCh, + } +} + +// Register adds a consumer to the Hub under the given topic and group. +// An empty groupID registers a universal (fan-out) consumer. +func (h *Hub) Register(id, topic, groupID string, ch chan *pb.QueueMessage) { + entry := &ConsumerEntry{ + ID: id, + Topic: topic, + Group: groupID, + Ch: ch, + } + + h.mu.Lock() + if h.topics[topic] == nil { + h.topics[topic] = newTopicSubscription() + } + h.topics[topic].add(entry) + h.byID[id] = entry + h.mu.Unlock() + + h.logger.Info("consumer registered", + zap.String("id", id), + zap.String("topic", topic), + zap.String("group", groupID), + zap.Bool("universal", entry.IsUniversal()), + ) + + // Wake the dispatcher loop immediately. + select { + case h.wakeCh <- struct{}{}: + default: + } +} + +// Unregister removes a consumer from the Hub and deletes its in-flight keys. +func (h *Hub) Unregister(id string) { + h.mu.Lock() + entry, ok := h.byID[id] + if !ok { + h.mu.Unlock() + return + } + + if sub, exists := h.topics[entry.Topic]; exists { + if sub.remove(entry) { + delete(h.topics, entry.Topic) + } + } + delete(h.byID, id) + h.mu.Unlock() + + h.logger.Info("consumer unregistered", + zap.String("id", id), + zap.String("topic", entry.Topic), + zap.String("group", entry.Group), + ) + + // Delete in-flight keys for this consumer. + h.inFlightMu.Lock() + delete(h.inFlightByConsumer, id) + h.inFlightMu.Unlock() +} + +// DispatchToTopic sends a message to all eligible consumers on a topic. +// For each group, the strategy selects one consumer. Universal consumers each +// receive a copy. Returns the group IDs the message was successfully sent to +// (universal consumers are reported with an empty group id). +func (h *Hub) DispatchToTopic(topic string, msg *pb.QueueMessage, deliveryTag []byte) []string { + h.mu.RLock() + sub, ok := h.topics[topic] + if !ok { + h.mu.RUnlock() + return nil + } + + // Snapshot groups and universal consumers while holding the lock. + groups := sub.groupSnapshot() + universal := sub.universalSnapshot() + h.mu.RUnlock() + + sentTo := make([]string, 0, len(groups)+len(universal)) + + // Dispatch to each group — strategy picks one consumer per group. + for gid, consumers := range groups { + selected := h.strategy.Select(consumers, msg) + if selected == nil { + continue + } + if h.trySend(selected, msg, deliveryTag) { + sentTo = append(sentTo, gid) + } + } + + // Dispatch to all universal consumers. + for _, c := range universal { + if h.trySend(c, msg, deliveryTag) { + sentTo = append(sentTo, "") + } + } + + return sentTo +} + +// trySend attempts to deliver a message to a single consumer. Returns true on +// success. Tracks the delivery tag as in-flight for the consumer. +func (h *Hub) trySend(c *ConsumerEntry, msg *pb.QueueMessage, deliveryTag []byte) bool { + select { + case c.Ch <- msg: + h.inFlightMu.Lock() + h.inFlightByConsumer[c.ID] = append(h.inFlightByConsumer[c.ID], deliveryTag) + h.inFlightMu.Unlock() + return true + default: + h.logger.Warn("consumer channel full, skipping", + zap.String("consumer_id", c.ID), + zap.String("topic", c.Topic), + zap.String("group", c.Group), + ) + return false + } +} + +// RemoveInFlightForConsumer removes a specific key from a consumer's in-flight +// tracking. Called when the consumer ACKs or NACKs a message. +func (h *Hub) RemoveInFlightForConsumer(consumerID string, key []byte) { + h.inFlightMu.Lock() + defer h.inFlightMu.Unlock() + keys := h.inFlightByConsumer[consumerID] + for i, k := range keys { + if bytes.Equal(k, key) { + h.inFlightByConsumer[consumerID] = append(keys[:i], keys[i+1:]...) + return + } + } +} + +// ActiveTopics returns a snapshot of all topics that currently have at least +// one connected consumer. +func (h *Hub) ActiveTopics() []string { + h.mu.RLock() + defer h.mu.RUnlock() + + result := make([]string, 0, len(h.topics)) + for topic := range h.topics { + result = append(result, topic) + } + return result +} + +// HasConsumers returns true if at least one consumer is currently connected. +func (h *Hub) HasConsumers() bool { + h.mu.RLock() + defer h.mu.RUnlock() + return len(h.byID) > 0 +} + +// GroupsForTopic returns a snapshot of all group IDs that have active consumers +// for the given topic. Does not include universal consumers. +func (h *Hub) GroupsForTopic(topic string) []string { + h.mu.RLock() + defer h.mu.RUnlock() + sub, ok := h.topics[topic] + if !ok { + return nil + } + result := make([]string, 0, len(sub.groups)) + for gid, consumers := range sub.groups { + if len(consumers) > 0 { + result = append(result, gid) + } + } + return result +} diff --git a/internal/dispatcher/hub_test.go b/internal/dispatcher/hub_test.go new file mode 100644 index 0000000..9d23928 --- /dev/null +++ b/internal/dispatcher/hub_test.go @@ -0,0 +1,301 @@ +package dispatcher + +import ( + "testing" + + pb "github.com/futureq-io/protocol/proto/go" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" +) + +type HubSuite struct { + suite.Suite +} + +func TestHubSuite(t *testing.T) { + suite.Run(t, new(HubSuite)) +} + +func (s *HubSuite) newHub(wakeCh chan struct{}) *Hub { + return NewHub(NewRoundRobinStrategy(), zap.NewNop(), wakeCh) +} + +func (s *HubSuite) newConsumer(id, topic, group string) *ConsumerEntry { + return &ConsumerEntry{ + ID: id, + Topic: topic, + Group: group, + Ch: make(chan *pb.QueueMessage, 16), + } +} + +// ─── ConsumerEntry ────────────────────────────────────────────────────────── + +func (s *HubSuite) TestConsumerEntry_GroupKey_UniversalUnique() { + require := s.Require() + + // Universal consumers (empty group) must each get a unique GroupKey. + c1 := &ConsumerEntry{ID: "id-1", Topic: "t", Group: ""} + c2 := &ConsumerEntry{ID: "id-2", Topic: "t", Group: ""} + require.NotEqual(c1.GroupKey(), c2.GroupKey()) +} + +func (s *HubSuite) TestConsumerEntry_GroupKey_SameGroup_SameKey() { + require := s.Require() + + c1 := &ConsumerEntry{ID: "a", Topic: "t", Group: "g1"} + c2 := &ConsumerEntry{ID: "b", Topic: "t", Group: "g1"} + require.Equal(c1.GroupKey(), c2.GroupKey()) +} + +func (s *HubSuite) TestConsumerEntry_IsUniversal() { + require := s.Require() + + require.True((&ConsumerEntry{Group: ""}).IsUniversal()) + require.False((&ConsumerEntry{Group: "g1"}).IsUniversal()) +} + +// ─── Register / Unregister ────────────────────────────────────────────────── + +func (s *HubSuite) TestRegister_SendsWakeSignal() { + require := s.Require() + + wakeCh := make(chan struct{}, 1) + h := s.newHub(wakeCh) + + h.Register("id-1", "topic", "group-1", make(chan *pb.QueueMessage, 1)) + + select { + case <-wakeCh: + // OK + default: + require.Fail("expected wake signal on Register") + } +} + +func (s *HubSuite) TestRegister_TopicListedInActiveTopics() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + h.Register("id-1", "orders", "g1", make(chan *pb.QueueMessage, 1)) + + topics := h.ActiveTopics() + require.Contains(topics, "orders") +} + +func (s *HubSuite) TestRegister_HasConsumers_True() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + require.False(h.HasConsumers()) + + h.Register("id-1", "orders", "g1", make(chan *pb.QueueMessage, 1)) + require.True(h.HasConsumers()) +} + +func (s *HubSuite) TestUnregister_RemovesFromActiveTopics() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + h.Register("id-1", "orders", "g1", make(chan *pb.QueueMessage, 1)) + h.Unregister("id-1") + + require.Empty(h.ActiveTopics()) + require.False(h.HasConsumers()) +} + +func (s *HubSuite) TestUnregister_UnknownID_NoOp() { + h := s.newHub(make(chan struct{}, 1)) + // Must not panic. + h.Unregister("nonexistent") +} + +// ─── DispatchToTopic ──────────────────────────────────────────────────────── + +func (s *HubSuite) TestDispatchToTopic_UnknownTopic_ReturnsZero() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + msg := &pb.QueueMessage{Topic: "nothing"} + + require.Empty(h.DispatchToTopic("nothing", msg, []byte("tag"))) +} + +func (s *HubSuite) TestDispatchToTopic_GroupedConsumer_ExactlyOneReceives() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + + ch1 := make(chan *pb.QueueMessage, 1) + ch2 := make(chan *pb.QueueMessage, 1) + h.Register("c1", "orders", "g1", ch1) + h.Register("c2", "orders", "g1", ch2) + + msg := &pb.QueueMessage{Topic: "orders", Payload: []byte("x")} + sent := h.DispatchToTopic("orders", msg, []byte("tag")) + require.Len(sent, 1, "only one consumer in the group should receive the message") + require.Equal("g1", sent[0]) + + // Exactly one of ch1, ch2 should have the message. + got1 := len(ch1) + got2 := len(ch2) + require.Equal(1, got1+got2) +} + +func (s *HubSuite) TestDispatchToTopic_UniversalConsumers_AllReceive() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + + ch1 := make(chan *pb.QueueMessage, 1) + ch2 := make(chan *pb.QueueMessage, 1) + // Empty group → universal. + h.Register("u1", "orders", "", ch1) + h.Register("u2", "orders", "", ch2) + + msg := &pb.QueueMessage{Topic: "orders", Payload: []byte("x")} + sent := h.DispatchToTopic("orders", msg, []byte("tag")) + require.Len(sent, 2, "each universal consumer should receive a copy") +} + +func (s *HubSuite) TestDispatchToTopic_MultipleGroups_EachGroupReceivesOne() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + + chG1A := make(chan *pb.QueueMessage, 1) + chG1B := make(chan *pb.QueueMessage, 1) + chG2 := make(chan *pb.QueueMessage, 1) + + h.Register("c1", "orders", "g1", chG1A) + h.Register("c2", "orders", "g1", chG1B) + h.Register("c3", "orders", "g2", chG2) + + msg := &pb.QueueMessage{Topic: "orders"} + sent := h.DispatchToTopic("orders", msg, []byte("tag")) + require.Len(sent, 2, "one consumer per group → 2 groups → 2 sends") + require.Contains(sent, "g1") + require.Contains(sent, "g2") +} + +func (s *HubSuite) TestDispatchToTopic_FullChannel_Skipped() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + + // Buffer of 1, fill it before dispatch. + ch := make(chan *pb.QueueMessage, 1) + ch <- &pb.QueueMessage{Topic: "other"} + + h.Register("c1", "orders", "g1", ch) + + msg := &pb.QueueMessage{Topic: "orders"} + sent := h.DispatchToTopic("orders", msg, []byte("tag")) + require.Empty(sent, "full channel should be skipped without blocking") +} + +// ─── RemoveInFlightForConsumer ────────────────────────────────────────────── + +func (s *HubSuite) TestRemoveInFlightForConsumer_RemovesMatchingKey() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + + ch := make(chan *pb.QueueMessage, 2) + h.Register("c1", "orders", "g1", ch) + + // Two successful dispatches → two keys in flight. + msg := &pb.QueueMessage{Topic: "orders"} + key1 := []byte("key-1") + key2 := []byte("key-2") + h.DispatchToTopic("orders", msg, key1) + h.DispatchToTopic("orders", msg, key2) + + require.Len(h.inFlightByConsumer["c1"], 2) + + h.RemoveInFlightForConsumer("c1", key1) + require.Len(h.inFlightByConsumer["c1"], 1) + require.Equal(key2, h.inFlightByConsumer["c1"][0]) +} + +func (s *HubSuite) TestRemoveInFlightForConsumer_UnknownConsumer_NoOp() { + h := s.newHub(make(chan struct{}, 1)) + h.RemoveInFlightForConsumer("nobody", []byte("key")) + // Must not panic. +} + +// ─── GroupsForTopic ───────────────────────────────────────────────────────── + +func (s *HubSuite) TestGroupsForTopic_OnlyGroupedConsumers() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + h.Register("c1", "orders", "g1", make(chan *pb.QueueMessage, 1)) + h.Register("c2", "orders", "g2", make(chan *pb.QueueMessage, 1)) + h.Register("u1", "orders", "", make(chan *pb.QueueMessage, 1)) // universal + + groups := h.GroupsForTopic("orders") + require.Len(groups, 2) + require.Contains(groups, "g1") + require.Contains(groups, "g2") +} + +func (s *HubSuite) TestGroupsForTopic_UnknownTopic_ReturnsNil() { + require := s.Require() + + h := s.newHub(make(chan struct{}, 1)) + require.Nil(h.GroupsForTopic("missing")) +} + +// ─── RoundRobinStrategy ────────────────────────────────────────────────────── + +func (s *HubSuite) TestRoundRobinStrategy_EmptyCandidates_ReturnsNil() { + require := s.Require() + + strategy := NewRoundRobinStrategy() + require.Nil(strategy.Select(nil, &pb.QueueMessage{})) +} + +func (s *HubSuite) TestRoundRobinStrategy_CyclesThroughConsumers() { + require := s.Require() + + strategy := NewRoundRobinStrategy() + + c1 := &ConsumerEntry{ID: "c1", Topic: "t", Group: "g"} + c2 := &ConsumerEntry{ID: "c2", Topic: "t", Group: "g"} + c3 := &ConsumerEntry{ID: "c3", Topic: "t", Group: "g"} + + candidates := []*ConsumerEntry{c1, c2, c3} + + // Call Select 6 times — each candidate should be picked exactly twice. + counts := make(map[string]int) + for i := 0; i < 6; i++ { + sel := strategy.Select(candidates, &pb.QueueMessage{}) + require.NotNil(sel) + counts[sel.ID]++ + } + + require.Equal(2, counts["c1"]) + require.Equal(2, counts["c2"]) + require.Equal(2, counts["c3"]) +} + +func (s *HubSuite) TestRoundRobinStrategy_PerGroupCountersAreIndependent() { + require := s.Require() + + strategy := NewRoundRobinStrategy() + + g1a := &ConsumerEntry{ID: "g1a", Topic: "t", Group: "g1"} + g1b := &ConsumerEntry{ID: "g1b", Topic: "t", Group: "g1"} + + g2a := &ConsumerEntry{ID: "g2a", Topic: "t", Group: "g2"} + g2b := &ConsumerEntry{ID: "g2b", Topic: "t", Group: "g2"} + + // First call in g1 → g1a; first call in g2 → g2a. + require.Equal("g1a", strategy.Select([]*ConsumerEntry{g1a, g1b}, nil).ID) + require.Equal("g2a", strategy.Select([]*ConsumerEntry{g2a, g2b}, nil).ID) + + // Second call in g1 → g1b; second in g2 → g2b. + require.Equal("g1b", strategy.Select([]*ConsumerEntry{g1a, g1b}, nil).ID) + require.Equal("g2b", strategy.Select([]*ConsumerEntry{g2a, g2b}, nil).ID) +} diff --git a/internal/dispatcher/janitor.go b/internal/dispatcher/janitor.go new file mode 100644 index 0000000..dbac7fd --- /dev/null +++ b/internal/dispatcher/janitor.go @@ -0,0 +1,111 @@ +package dispatcher + +import ( + "context" + "time" + + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + + "github.com/futureq-io/futureq/internal/metrics" + "github.com/futureq-io/futureq/internal/storage" + "github.com/futureq-io/futureq/pkg/utils" + storagepb "github.com/futureq-io/protocol/proto/go/storage" +) + +// TTLJanitor periodically performs a full storage scan and removes messages +// whose TTL has elapsed. Unlike the dispatcher (which only scans active-topic +// ranges), the janitor sweeps all keys so expired messages are cleaned up even +// when no consumer is connected. +// +// Expired keys are forwarded to the Deleter, which routes them through Raft +// (or the local storage engine directly in single-node mode) as a batched +// DeleteBatchCmd. +type TTLJanitor struct { + db storage.DB + deleter *Deleter + interval time.Duration + logger *zap.Logger +} + +// NewTTLJanitor constructs a TTLJanitor. interval controls how often the full +// scan runs (e.g., 60 seconds). Shorter intervals mean faster cleanup at the +// cost of more I/O. +func NewTTLJanitor(db storage.DB, deleter *Deleter, interval time.Duration, logger *zap.Logger) *TTLJanitor { + return &TTLJanitor{ + db: db, + deleter: deleter, + interval: interval, + logger: logger.Named("ttl_janitor"), + } +} + +// Run starts the TTL janitor loop. It blocks until ctx is cancelled. +func (j *TTLJanitor) Run(ctx context.Context) { + // Run the first pass after one full interval to avoid startup contention. + ticker := time.NewTicker(j.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + j.sweep() + } + } +} + +// sweep performs one full scan of the storage engine and collects expired +// message keys. Uses Scan for lower overhead — no manual iterator lifecycle. +func (j *TTLJanitor) sweep() { + nowMs := time.Now().UnixMilli() + var expiredKeys [][]byte + + err := j.db.Scan(nil, func(key, val []byte) error { + // Only consider 24-byte event keys. + if _, _, _, err := utils.ParseEventKey(key); err != nil { + // Not an event key (metadata, indexes, etc.) — skip silently. + return nil + } + + var msg storagepb.StoredMessage + if err := proto.Unmarshal(val, &msg); err != nil { + // Skip keys we can't parse. + return nil + } + + if msg.TtlMs <= 0 { + // No TTL set — message lives forever. + return nil + } + + if nowMs >= msg.EnqueuedAtUnixMs+msg.TtlMs { + // Must copy — key is only valid during yield. + keyCopy := make([]byte, len(key)) + copy(keyCopy, key) + expiredKeys = append(expiredKeys, keyCopy) + metrics.MessagesExpiredTotal.WithLabelValues(msg.Topic, "janitor").Inc() + } + + return nil + }) + + if err != nil { + j.logger.Error("TTL janitor: scan error", zap.Error(err)) + return + } + + if len(expiredKeys) == 0 { + return + } + + // Enqueue expired keys for batched deletion via the Deleter. + for _, k := range expiredKeys { + j.deleter.MarkDeleted(k) + } + + j.logger.Info("TTL janitor: marked expired messages for deletion", + zap.Int("count", len(expiredKeys)), + ) +} diff --git a/internal/dispatcher/janitor_test.go b/internal/dispatcher/janitor_test.go new file mode 100644 index 0000000..5f27776 --- /dev/null +++ b/internal/dispatcher/janitor_test.go @@ -0,0 +1,225 @@ +package dispatcher + +import ( + "context" + "testing" + "time" + + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/storage" + "github.com/futureq-io/futureq/pkg/utils" + storagepb "github.com/futureq-io/protocol/proto/go/storage" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" +) + +type JanitorSuite struct { + suite.Suite + db storage.DB +} + +func TestJanitorSuite(t *testing.T) { + suite.Run(t, new(JanitorSuite)) +} + +func (s *JanitorSuite) SetupTest() { + db, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + s.Require().NoError(err) + s.db = db +} + +func (s *JanitorSuite) TearDownTest() { + if s.db != nil { + s.db.Close() + } +} + +// makeEventKey constructs a 24-byte event key. +func makeEventKey(topic string, bucket, eventID uint64) []byte { + return utils.EventKey(bucket, utils.TopicHash(topic), eventID) +} + +// storeMessage writes a marshalled StoredMessage under the given key. +func (s *JanitorSuite) storeMessage(key []byte, msg *storagepb.StoredMessage) { + data, err := proto.Marshal(msg) + s.Require().NoError(err) + + b := s.db.NewBatch() + s.Require().NoError(b.Set(key, data)) + s.Require().NoError(b.Commit(storage.Sync)) + s.Require().NoError(b.Close()) +} + +// ─── TTLJanitor.sweep ─────────────────────────────────────────────────────── + +func (s *JanitorSuite) TestSweep_MarksExpiredMessages() { + require := s.Require() + + nowMs := time.Now().UnixMilli() + + expiredKey := makeEventKey("orders", 1, 1) + s.storeMessage(expiredKey, &storagepb.StoredMessage{ + Topic: "orders", + EnqueuedAtUnixMs: nowMs - 10_000, + TtlMs: 5_000, // expired 5s ago + }) + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + janitor := NewTTLJanitor(s.db, d, time.Hour, zap.NewNop()) + + janitor.sweep() + + d.mu.Lock() + defer d.mu.Unlock() + require.Len(d.pending, 1) + require.Equal(expiredKey, d.pending[0]) +} + +func (s *JanitorSuite) TestSweep_SkipsNonExpiredMessages() { + require := s.Require() + + nowMs := time.Now().UnixMilli() + + liveKey := makeEventKey("orders", 1, 1) + s.storeMessage(liveKey, &storagepb.StoredMessage{ + Topic: "orders", + EnqueuedAtUnixMs: nowMs, + TtlMs: 60_000, // live for another minute + }) + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + janitor := NewTTLJanitor(s.db, d, time.Hour, zap.NewNop()) + + janitor.sweep() + + d.mu.Lock() + defer d.mu.Unlock() + require.Empty(d.pending) +} + +func (s *JanitorSuite) TestSweep_SkipsNoTTL() { + require := s.Require() + + nowMs := time.Now().UnixMilli() + + noTTLKey := makeEventKey("orders", 1, 1) + s.storeMessage(noTTLKey, &storagepb.StoredMessage{ + Topic: "orders", + EnqueuedAtUnixMs: nowMs - 1_000_000, // very old, but no TTL + TtlMs: 0, + }) + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + janitor := NewTTLJanitor(s.db, d, time.Hour, zap.NewNop()) + + janitor.sweep() + + d.mu.Lock() + defer d.mu.Unlock() + require.Empty(d.pending) +} + +func (s *JanitorSuite) TestSweep_SkipsNonEventKeys() { + require := s.Require() + + // A metadata-style key (not 24 bytes) should be silently skipped. + b := s.db.NewBatch() + require.NoError(b.Set([]byte("metadata/some-other-key"), []byte("not-proto"))) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + janitor := NewTTLJanitor(s.db, d, time.Hour, zap.NewNop()) + + janitor.sweep() + + d.mu.Lock() + defer d.mu.Unlock() + require.Empty(d.pending) +} + +func (s *JanitorSuite) TestSweep_SkipsUnparseableValue() { + require := s.Require() + + // Valid 24-byte key but value is not a valid protobuf. + key := makeEventKey("orders", 1, 1) + b := s.db.NewBatch() + require.NoError(b.Set(key, []byte{0xFF, 0xFE, 0xFD})) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + janitor := NewTTLJanitor(s.db, d, time.Hour, zap.NewNop()) + + janitor.sweep() + + d.mu.Lock() + defer d.mu.Unlock() + require.Empty(d.pending) +} + +func (s *JanitorSuite) TestSweep_MultipleExpired_AllMarked() { + require := s.Require() + + nowMs := time.Now().UnixMilli() + + keys := [][]byte{ + makeEventKey("t1", 1, 1), + makeEventKey("t2", 2, 2), + makeEventKey("t3", 3, 3), + } + for _, k := range keys { + s.storeMessage(k, &storagepb.StoredMessage{ + Topic: "t", + EnqueuedAtUnixMs: nowMs - 100_000, + TtlMs: 1_000, + }) + } + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + janitor := NewTTLJanitor(s.db, d, time.Hour, zap.NewNop()) + + janitor.sweep() + + d.mu.Lock() + defer d.mu.Unlock() + require.Len(d.pending, 3) +} + +// ─── TTLJanitor.Run ───────────────────────────────────────────────────────── + +func (s *JanitorSuite) TestRun_SweepsOnInterval() { + require := s.Require() + + nowMs := time.Now().UnixMilli() + expiredKey := makeEventKey("t", 1, 1) + s.storeMessage(expiredKey, &storagepb.StoredMessage{ + Topic: "t", + EnqueuedAtUnixMs: nowMs - 60_000, + TtlMs: 1_000, + }) + + backend := &mockBackend{} + d := NewDeleter(backend, time.Hour, zap.NewNop()) + janitor := NewTTLJanitor(s.db, d, 10*time.Millisecond, zap.NewNop()) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { janitor.Run(ctx); close(done) }() + + require.Eventually(func() bool { + d.mu.Lock() + defer d.mu.Unlock() + return len(d.pending) > 0 + }, time.Second, 5*time.Millisecond) + + cancel() + <-done +} diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go new file mode 100644 index 0000000..d7ab668 --- /dev/null +++ b/internal/metrics/prometheus.go @@ -0,0 +1,175 @@ +package metrics + +import ( + "context" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" +) + +// All histograms use millisecond units. +var ( + // ─── Producer ──────────────────────────────────────────────────────────── + + // PublishRequestsTotal counts every PublishBatch frame processed, + // tagged by outcome so success/error rates can be computed. + PublishRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "futureq_publish_requests_total", + Help: "Total publish batch requests, partitioned by outcome.", + }, []string{"topic", "ack_level", "result"}) + + // MessagesPublishedTotal counts individual messages (not batches). + MessagesPublishedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "futureq_messages_published_total", + Help: "Total number of messages successfully published.", + }, []string{"topic", "ack_level"}) + + // PublishBatchSize records the distribution of batch sizes. + PublishBatchSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "futureq_publish_batch_size", + Help: "Number of messages in each publish batch.", + Buckets: prometheus.ExponentialBuckets(1, 2, 12), // 1..4096 + }, []string{"topic"}) + + // PublishLatencyMs measures the full per-batch processing time on the + // broker, from receiving the frame to just before sending the ack. + PublishLatencyMs = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "futureq_publish_latency_ms", + Help: "Broker-side publish latency per batch in milliseconds.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 16), // 0.1ms..~6.5s + }, []string{"topic", "ack_level"}) + + // RaftProposeDurationMs measures just the Raft SyncPropose / Propose call. + RaftProposeDurationMs = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "futureq_raft_propose_duration_ms", + Help: "Latency of Raft SyncPropose calls in milliseconds.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 16), // 0.1ms..~6.5s + }, []string{"ack_level"}) + + // ─── Consumer ──────────────────────────────────────────────────────────── + + // ActiveConsumers tracks currently connected consumers. + ActiveConsumers = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "futureq_active_consumers", + Help: "Current number of connected consumers.", + }, []string{"topic", "group_id"}) + + // ConsumerAckTotal counts ACK (success=true) and NACK (success=false) frames. + ConsumerAckTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "futureq_consumer_ack_total", + Help: "Total number of consumer acknowledgements received.", + }, []string{"topic", "group_id", "success"}) + + // MessagesInFlight tracks messages dispatched but not yet ACKed/NACKed. + MessagesInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "futureq_messages_in_flight", + Help: "Current number of dispatched but unacknowledged messages.", + }, []string{"topic", "group_id"}) + + // ─── Dispatcher / delivery ─────────────────────────────────────────────── + + // MessagesDispatchedTotal counts each successful send to a consumer channel. + // For fan-out topics the same message is counted once per recipient. + MessagesDispatchedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "futureq_messages_dispatched_total", + Help: "Total number of messages dispatched to consumers.", + }, []string{"topic", "group_id"}) + + // DispatchPassDurationMs measures one full dispatcher scan pass. + DispatchPassDurationMs = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "futureq_dispatch_pass_duration_ms", + Help: "Duration of each dispatcher scan pass in milliseconds.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 16), + }) + + // DeliveryLatencyMs measures the total time from when the producer enqueued + // the message to when the dispatcher handed it to a consumer. Includes any + // intentional DelayMs the producer requested. + DeliveryLatencyMs = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "futureq_delivery_latency_ms", + Help: "Enqueue-to-dispatch latency in milliseconds (includes producer-requested delay).", + Buckets: prometheus.ExponentialBuckets(1, 2, 16), // 1ms..~65s + }, []string{"topic"}) + + // DeliveryOverheadMs measures how late the dispatch was relative to the + // message's scheduled delivery time (EnqueuedAt + DelayMs). For messages + // with no delay this equals DeliveryLatencyMs. A rising p99 here means the + // dispatcher is falling behind. + DeliveryOverheadMs = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "futureq_delivery_overhead_ms", + Help: "Dispatch lateness past scheduled delivery time in milliseconds.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 16), // 0.1ms..~6.5s + }, []string{"topic"}) + + // MessagesExpiredTotal counts messages discarded because their TTL elapsed. + // source = "dispatcher" or "janitor". + MessagesExpiredTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "futureq_messages_expired_total", + Help: "Total number of messages discarded due to TTL expiry.", + }, []string{"topic", "source"}) + + // ─── Deleter ───────────────────────────────────────────────────────────── + + // DeleteBatchSize records the distribution of batched deletes. + DeleteBatchSize = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "futureq_delete_batch_size", + Help: "Number of keys in each batched delete flush.", + Buckets: prometheus.ExponentialBuckets(1, 2, 12), + }) + + // DeleteFailuresTotal counts failed batched delete attempts (will be retried). + DeleteFailuresTotal = promauto.NewCounter(prometheus.CounterOpts{ + Name: "futureq_delete_failures_total", + Help: "Total failed batched delete flushes.", + }) +) + +// Server wraps the Prometheus HTTP metrics server. +type Server struct { + addr string + logger *zap.Logger +} + +// NewServer creates a metrics HTTP server that will expose /metrics. +// addr should be in the form "host:port" (e.g. "0.0.0.0:9090"). +func NewServer(addr string, logger *zap.Logger) *Server { + return &Server{ + addr: addr, + logger: logger.Named("metrics"), + } +} + +// Run starts the HTTP server and blocks until ctx is cancelled. +func (s *Server) Run(ctx context.Context) { + if s.addr == "" { + s.logger.Info("metrics server disabled (no listen address configured)") + <-ctx.Done() + return + } + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + + srv := &http.Server{ + Addr: s.addr, + Handler: mux, + } + + go func() { + s.logger.Info("metrics server listening", zap.String("address", s.addr)) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + s.logger.Error("metrics server error", zap.Error(err)) + } + }() + + <-ctx.Done() + s.logger.Info("metrics server: shutting down") + _ = srv.Shutdown(context.Background()) //nolint:contextcheck +} diff --git a/internal/q/q.go b/internal/q/q.go deleted file mode 100644 index 0d251db..0000000 --- a/internal/q/q.go +++ /dev/null @@ -1,8 +0,0 @@ -package q - -type Q interface { - Connect() error - Consume() error - Publish(payload []byte) - Close() -} diff --git a/internal/q/rabbitmq.go b/internal/q/rabbitmq.go deleted file mode 100644 index 0c2c9a4..0000000 --- a/internal/q/rabbitmq.go +++ /dev/null @@ -1,172 +0,0 @@ -package q - -import ( - "context" - "errors" - "fmt" - "strconv" - "time" - - amqp "github.com/rabbitmq/amqp091-go" - "go.uber.org/zap" - - "github.com/futureq-io/futureq/internal/config" - "github.com/futureq-io/futureq/internal/storage" -) - -type rabbitMQ struct { - cfg config.RabbitMQ - logger *zap.Logger - rabbitConn *amqp.Connection - rabbitChan *amqp.Channel - storage storage.TaskStorage -} - -func NewRabbitMQ(cfg config.RabbitMQ, logger *zap.Logger, taskStorage storage.TaskStorage) Q { - return &rabbitMQ{ - cfg: cfg, - logger: logger, - storage: taskStorage, - } -} - -func (r *rabbitMQ) Connect() error { - var err error - - r.rabbitConn, err = amqp.Dial(r.cfg.RabbitMQServer.ConnectionURI()) - if err != nil { - return fmt.Errorf("error in connecting to rabbitmq: %w", err) - } - - r.rabbitChan, err = r.rabbitConn.Channel() - if err != nil { - return fmt.Errorf("error in creating channel to rabbitmq: %w", err) - } - - return nil -} - -func (r *rabbitMQ) Consume() error { - if r.cfg.RabbitMQDataExchange.DeclareQueue { - _, err := r.rabbitChan.QueueDeclare( - r.cfg.RabbitMQDataExchange.ConsumeQueueName, - false, - false, - false, - false, - nil, - ) - - if err != nil { - return fmt.Errorf("error in declaring queue: %w", err) - } - } - - deliveryChan, err := r.rabbitChan.Consume( - r.cfg.RabbitMQDataExchange.ConsumeQueueName, - "", - true, - false, - false, - false, - nil, - ) - if err != nil { - return fmt.Errorf("error in consuming queue: %w", err) - } - - go r.consumeLoop(deliveryChan, r.cfg.RabbitMQDataExchange.ConsumeQueueName) - - return nil -} - -func (r *rabbitMQ) consumeLoop(deliveryChan <-chan amqp.Delivery, queue string) { - for delivery := range deliveryChan { - - startedAt := time.Now() - err := r.handleDelivery(delivery) - duration := time.Since(startedAt) - - log := r.logger.With( - zap.String("exchange", delivery.Exchange), - zap.String("queue", queue), - zap.String("routing_key", delivery.RoutingKey), - zap.String("consumer_tag", delivery.ConsumerTag), - zap.Uint64("delivery_tag", delivery.DeliveryTag), - zap.String("message_id", delivery.MessageId), - zap.String("user_id", delivery.UserId), - zap.String("app_id", delivery.AppId), - zap.Error(err), - zap.String("duration", duration.String()), - ) - if err != nil { - log.Error("error in processing message") - } else { - log.Debug("message processed successfully") - } - } -} - -func (r *rabbitMQ) handleDelivery(delivery amqp.Delivery) error { - xFutureReceiveAtVal, ok := delivery.Headers[xFutureReceiveAtHeader] - if !ok { - return ErrReceivedAtHeaderNotExists - } - - var receivedAt time.Time - if xFutureReceiveAtInt64, ok := xFutureReceiveAtVal.(int64); ok { - receivedAt = time.UnixMilli(xFutureReceiveAtInt64) - } else if xFutureReceiveAtUInt64, ok := xFutureReceiveAtVal.(uint64); ok { - receivedAt = time.UnixMilli(int64(xFutureReceiveAtUInt64)) - } else if xFutureReceiveAtString, ok := xFutureReceiveAtVal.(string); ok { - xFutureReceiveAtParsed, err := strconv.ParseInt(xFutureReceiveAtString, 10, 64) - if err != nil { - return ErrInvalidReceivedAtHeaderFormat - } - - receivedAt = time.UnixMilli(xFutureReceiveAtParsed) - } else { - return ErrInvalidReceivedAtHeaderFormat - } - - r.storage.Add(delivery.Body, receivedAt) - - return nil -} - -func (r *rabbitMQ) Publish(payload []byte) { - err := r.rabbitChan.PublishWithContext(context.TODO(), - "", - r.cfg.RabbitMQDataExchange.ProduceQueueName, - false, - false, - amqp.Publishing{ - DeliveryMode: amqp.Persistent, - ContentType: "text/plain", - Body: payload, - }) - if err != nil { - r.logger.Error("Failed to publish a message", zap.Error(err)) - } -} - -func (r *rabbitMQ) Close() { - err := r.rabbitChan.Close() - if err != nil { - r.logger.Error("Failed to close rabbitMQ channel", zap.Error(err)) - } - - err = r.rabbitConn.Close() - if err != nil { - r.logger.Error("Failed to close rabbitMQ connection", zap.Error(err)) - } -} - -const ( - xFutureReceiveAtHeader = "x-future-deliver-at" -) - -var ( - ErrReceivedAtHeaderNotExists = errors.New("received at header does not exist") - ErrInvalidReceivedAtHeaderFormat = errors.New("invalid received at header format") -) diff --git a/internal/raft/event/commands.go b/internal/raft/event/commands.go new file mode 100644 index 0000000..dfac025 --- /dev/null +++ b/internal/raft/event/commands.go @@ -0,0 +1,205 @@ +package raft + +import ( + "encoding/binary" + "fmt" +) + +// CommandType identifies the state machine operation. +type CommandType uint8 + +const ( + // StoreBatchCmd atomically writes a batch of messages to Pebble. + // The state machine uses the shared EventRepository to assign keys, + // ensuring consistent monotonic IDs across both Raft and standalone modes. + StoreBatchCmd CommandType = iota + + // DeleteBatchCmd atomically deletes a batch of message keys from Pebble. + // Used for Raft-replicated ACK-driven deletions and TTL expirations. + DeleteBatchCmd +) + +// StoreBatchItem is the minimal per-message metadata carried in a StoreBatchCmd. +// The serialised StoredMessage value is passed verbatim — no re-serialisation +// occurs in the state machine. The state machine calls EventRepository to assign +// the monotonic event ID and construct the full 24-byte key. +type StoreBatchItem struct { + // Bucket is the pre-computed time bucket (fire_at_ms / timeBucketSize). + Bucket uint64 + // TopicHash is xxhash64(topic). + TopicHash uint64 + Indexes [][]byte + // Msg is the already-serialised storagepb.StoredMessage proto bytes. + // This slice aliases the original command buffer — do not mutate. + Msg []byte +} + +// MarshalStoreBatchCmd serialises a list of items into a compact binary command. +// +// Wire format: +// +// [0] CommandType (1 byte = 0) +// [1..8] count (uint64 big-endian) +// for each item: +// [n..n+7] bucket (uint64 big-endian) +// [n+8..n+15] topicHash (uint64 big-endian) +// [n+16..n+17] numIdx (uint16 big-endian) +// for each index: +// [m..m+1] idxLen (uint16 big-endian) +// [m+2..] idxData (idxLen bytes) +// [x..x+3] valLen (uint32 big-endian) +// [x+4..] value (valLen bytes — serialised StoredMessage) +// +// The Msg and Indexes slices in items are appended directly with a single copy. +func MarshalStoreBatchCmd(items []StoreBatchItem) ([]byte, error) { + // Pre-calculate total size to do a single allocation. + size := 1 + 8 // cmdType + count + for _, it := range items { + size += 8 + 8 + 2 // bucket + topicHash + numIdx + for _, idx := range it.Indexes { + size += 2 + len(idx) // idxLen + data + } + size += 4 + len(it.Msg) // valLen + value + } + + out := make([]byte, size) + out[0] = byte(StoreBatchCmd) + binary.BigEndian.PutUint64(out[1:9], uint64(len(items))) + + pos := 9 + for _, it := range items { + binary.BigEndian.PutUint64(out[pos:pos+8], it.Bucket) + pos += 8 + binary.BigEndian.PutUint64(out[pos:pos+8], it.TopicHash) + pos += 8 + + binary.BigEndian.PutUint16(out[pos:pos+2], uint16(len(it.Indexes))) + pos += 2 + for _, idx := range it.Indexes { + binary.BigEndian.PutUint16(out[pos:pos+2], uint16(len(idx))) + pos += 2 + copy(out[pos:], idx) + pos += len(idx) + } + + binary.BigEndian.PutUint32(out[pos:pos+4], uint32(len(it.Msg))) + pos += 4 + copy(out[pos:], it.Msg) + pos += len(it.Msg) + } + + return out, nil +} + +// UnmarshalStoreBatchCmd deserialises a StoreBatchCmd payload. +// The Msg and Indexes slices in the returned items alias the input data slice — they are +// zero-copy views into the Dragonboat log buffer. Callers must not mutate them. +func UnmarshalStoreBatchCmd(data []byte) ([]StoreBatchItem, error) { + if len(data) < 1+8 { + return nil, fmt.Errorf("raft: StoreBatchCmd too short: %d bytes", len(data)) + } + if CommandType(data[0]) != StoreBatchCmd { + return nil, fmt.Errorf("raft: expected StoreBatchCmd (0), got %d", data[0]) + } + + count := binary.BigEndian.Uint64(data[1:9]) + items := make([]StoreBatchItem, 0, count) + + pos := 9 + for i := uint64(0); i < count; i++ { + if pos+8+8+2 > len(data) { + return nil, fmt.Errorf("raft: StoreBatchCmd truncated at item %d header", i) + } + bucket := binary.BigEndian.Uint64(data[pos : pos+8]) + pos += 8 + topicHash := binary.BigEndian.Uint64(data[pos : pos+8]) + pos += 8 + + numIdx := int(binary.BigEndian.Uint16(data[pos : pos+2])) + pos += 2 + + indexes := make([][]byte, 0, numIdx) + for j := 0; j < numIdx; j++ { + if pos+2 > len(data) { + return nil, fmt.Errorf("raft: StoreBatchCmd truncated at item %d index %d len", i, j) + } + idxLen := int(binary.BigEndian.Uint16(data[pos : pos+2])) + pos += 2 + + if pos+idxLen > len(data) { + return nil, fmt.Errorf("raft: StoreBatchCmd truncated at item %d index %d data", i, j) + } + indexes = append(indexes, data[pos : pos+idxLen]) + pos += idxLen + } + + if pos+4 > len(data) { + return nil, fmt.Errorf("raft: StoreBatchCmd truncated at item %d valLen", i) + } + valLen := int(binary.BigEndian.Uint32(data[pos : pos+4])) + pos += 4 + + if pos+valLen > len(data) { + return nil, fmt.Errorf("raft: StoreBatchCmd truncated at item %d value (need %d, have %d)", i, valLen, len(data)-pos) + } + // Zero-copy: alias the command buffer directly. + value := data[pos : pos+valLen] + pos += valLen + + items = append(items, StoreBatchItem{ + Bucket: bucket, + TopicHash: topicHash, + Indexes: indexes, + Msg: value, + }) + } + + return items, nil +} + +// MarshalDeleteBatchCmd serialises a list of 24-byte Pebble keys to delete. +// +// Wire format: +// +// [0] CommandType (1 byte = 1) +// [1..8] count (uint64 big-endian) +// for each key: +// [n..n+23] key (24 bytes — fixed size) +func MarshalDeleteBatchCmd(keys [][]byte) ([]byte, error) { + out := make([]byte, 1+8+len(keys)*24) + out[0] = byte(DeleteBatchCmd) + binary.BigEndian.PutUint64(out[1:9], uint64(len(keys))) + + pos := 9 + for _, k := range keys { + if len(k) != 24 { + return nil, fmt.Errorf("raft: DeleteBatchCmd: key must be 24 bytes, got %d", len(k)) + } + copy(out[pos:pos+24], k) + pos += 24 + } + + return out, nil +} + +// UnmarshalDeleteBatchCmd deserialises a DeleteBatchCmd payload. +// The returned key slices alias the input data — do not mutate data. +func UnmarshalDeleteBatchCmd(data []byte) ([][]byte, error) { + if len(data) < 1+8 { + return nil, fmt.Errorf("raft: DeleteBatchCmd too short: %d bytes", len(data)) + } + if CommandType(data[0]) != DeleteBatchCmd { + return nil, fmt.Errorf("raft: expected DeleteBatchCmd (1), got %d", data[0]) + } + count := int(binary.BigEndian.Uint64(data[1:9])) + if len(data) < 9+count*24 { + return nil, fmt.Errorf("raft: DeleteBatchCmd too short for %d keys", count) + } + + keys := make([][]byte, count) + for i := 0; i < count; i++ { + start := 9 + i*24 + keys[i] = data[start : start+24] + } + return keys, nil +} diff --git a/internal/raft/event/commands_test.go b/internal/raft/event/commands_test.go new file mode 100644 index 0000000..f9b8609 --- /dev/null +++ b/internal/raft/event/commands_test.go @@ -0,0 +1,183 @@ +package raft + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type CommandsSuite struct { + suite.Suite +} + +func TestCommandsSuite(t *testing.T) { + suite.Run(t, new(CommandsSuite)) +} + +// ─── StoreBatchCmd round-trips ────────────────────────────────────────────── + +func (s *CommandsSuite) TestStoreBatchCmd_BasicRoundTrip() { + require := s.Require() + + items := []StoreBatchItem{ + {Bucket: 100, TopicHash: 42, Msg: []byte("hello")}, + {Bucket: 200, TopicHash: 43, Msg: []byte("world")}, + } + + cmd, err := MarshalStoreBatchCmd(items) + require.NoError(err) + require.Equal(byte(StoreBatchCmd), cmd[0]) + + got, err := UnmarshalStoreBatchCmd(cmd) + require.NoError(err) + require.Len(got, 2) + require.Equal(uint64(100), got[0].Bucket) + require.Equal(uint64(42), got[0].TopicHash) + require.Equal([]byte("hello"), got[0].Msg) + require.Equal(uint64(200), got[1].Bucket) + require.Equal(uint64(43), got[1].TopicHash) + require.Equal([]byte("world"), got[1].Msg) +} + +func (s *CommandsSuite) TestStoreBatchCmd_WithIndexes() { + require := s.Require() + + items := []StoreBatchItem{ + { + Bucket: 5, + TopicHash: 99, + Indexes: [][]byte{[]byte("idx1"), []byte("idx2")}, + Msg: []byte("msg"), + }, + } + + cmd, err := MarshalStoreBatchCmd(items) + require.NoError(err) + + got, err := UnmarshalStoreBatchCmd(cmd) + require.NoError(err) + require.Len(got, 1) + require.Len(got[0].Indexes, 2) + require.Equal([]byte("idx1"), got[0].Indexes[0]) + require.Equal([]byte("idx2"), got[0].Indexes[1]) +} + +func (s *CommandsSuite) TestStoreBatchCmd_EmptyItemList() { + require := s.Require() + + cmd, err := MarshalStoreBatchCmd([]StoreBatchItem{}) + require.NoError(err) + + got, err := UnmarshalStoreBatchCmd(cmd) + require.NoError(err) + require.Empty(got) +} + +func (s *CommandsSuite) TestStoreBatchCmd_EmptyMsg() { + require := s.Require() + + items := []StoreBatchItem{{Bucket: 1, TopicHash: 1, Msg: []byte{}}} + cmd, err := MarshalStoreBatchCmd(items) + require.NoError(err) + + got, err := UnmarshalStoreBatchCmd(cmd) + require.NoError(err) + require.Len(got, 1) + require.Empty(got[0].Msg) +} + +func (s *CommandsSuite) TestStoreBatchCmd_TooShort_Fails() { + require := s.Require() + + _, err := UnmarshalStoreBatchCmd([]byte{byte(StoreBatchCmd), 0x01}) + require.Error(err) +} + +func (s *CommandsSuite) TestStoreBatchCmd_WrongType_Fails() { + require := s.Require() + + deleteCmd, _ := MarshalDeleteBatchCmd([][]byte{make([]byte, 24)}) + + _, err := UnmarshalStoreBatchCmd(deleteCmd) + require.Error(err) +} + +func (s *CommandsSuite) TestStoreBatchCmd_TruncatedMsg_Fails() { + require := s.Require() + + items := []StoreBatchItem{{Bucket: 1, TopicHash: 1, Msg: []byte("hello-world")}} + cmd, _ := MarshalStoreBatchCmd(items) + + // Cut buffer before end of msg. + _, err := UnmarshalStoreBatchCmd(cmd[:len(cmd)-3]) + require.Error(err) +} + +// ─── DeleteBatchCmd round-trips ────────────────────────────────────────────── + +func (s *CommandsSuite) TestDeleteBatchCmd_BasicRoundTrip() { + require := s.Require() + + keys := [][]byte{ + make([]byte, 24), + make([]byte, 24), + } + // Make keys distinct. + keys[0][0] = 0xAA + keys[1][0] = 0xBB + + cmd, err := MarshalDeleteBatchCmd(keys) + require.NoError(err) + require.Equal(byte(DeleteBatchCmd), cmd[0]) + + got, err := UnmarshalDeleteBatchCmd(cmd) + require.NoError(err) + require.Len(got, 2) + require.Equal(keys[0], got[0]) + require.Equal(keys[1], got[1]) +} + +func (s *CommandsSuite) TestDeleteBatchCmd_InvalidKeyLength_Fails() { + require := s.Require() + + _, err := MarshalDeleteBatchCmd([][]byte{[]byte("short")}) + require.Error(err) +} + +func (s *CommandsSuite) TestDeleteBatchCmd_EmptyList() { + require := s.Require() + + cmd, err := MarshalDeleteBatchCmd([][]byte{}) + require.NoError(err) + + got, err := UnmarshalDeleteBatchCmd(cmd) + require.NoError(err) + require.Empty(got) +} + +func (s *CommandsSuite) TestDeleteBatchCmd_TooShort_Fails() { + require := s.Require() + + _, err := UnmarshalDeleteBatchCmd([]byte{byte(DeleteBatchCmd), 0x00}) + require.Error(err) +} + +func (s *CommandsSuite) TestDeleteBatchCmd_WrongType_Fails() { + require := s.Require() + + storeCmd, _ := MarshalStoreBatchCmd([]StoreBatchItem{{Bucket: 1, TopicHash: 1, Msg: []byte("x")}}) + + _, err := UnmarshalDeleteBatchCmd(storeCmd) + require.Error(err) +} + +func (s *CommandsSuite) TestDeleteBatchCmd_TruncatedKeyData_Fails() { + require := s.Require() + + keys := [][]byte{make([]byte, 24)} + cmd, _ := MarshalDeleteBatchCmd(keys) + + // Cut mid-key. + _, err := UnmarshalDeleteBatchCmd(cmd[:len(cmd)-10]) + require.Error(err) +} diff --git a/internal/raft/event/statemachine.go b/internal/raft/event/statemachine.go new file mode 100644 index 0000000..2d6fe24 --- /dev/null +++ b/internal/raft/event/statemachine.go @@ -0,0 +1,287 @@ +package raft + +import ( + "encoding/binary" + "errors" + "io" + "log" + + "github.com/cockroachdb/pebble/v2" + "github.com/futureq-io/futureq/internal/repository" + "github.com/futureq-io/futureq/internal/storage" + "github.com/lni/dragonboat/v4/statemachine" + "go.uber.org/zap" +) + +var appliedIndexKey = []byte("metadata/raft/applied-index") + +// EventStateMachine implements statemachine.IOnDiskStateMachine. +// Pebble is used as the durable backing store; its WAL is intentionally +// disabled in clustered mode because the Dragonboat Raft log acts as the +// authoritative write-ahead log. On restart, Dragonboat replays any log +// entries that were committed but not yet applied, so no data is lost. +type EventStateMachine struct { + clusterID uint64 + nodeID uint64 + db storage.DB + repo *repository.EventRepository + lastApplied uint64 + // OnDeleteKeys is called after a DeleteBatchCmd is applied, with copies + // of each deleted key. Used to remove entries from the dispatcher's in-flight + // map. Safe to be nil. + OnDeleteKeys func(keys [][]byte) +} + +// NewEventStateMachineFactory returns the factory function that Dragonboat +// passes (clusterID, nodeID) to when it instantiates a new replica. +func NewEventStateMachineFactory(db storage.DB, repo *repository.EventRepository, onDeleteKeys func(keys [][]byte), logger *zap.Logger) func(uint64, uint64) statemachine.IOnDiskStateMachine { + return func(clusterID uint64, nodeID uint64) statemachine.IOnDiskStateMachine { + _ = logger + return &EventStateMachine{ + clusterID: clusterID, + nodeID: nodeID, + db: db, + repo: repo, + OnDeleteKeys: onDeleteKeys, + } + } +} + +func (s *EventStateMachine) Open(stopc <-chan struct{}) (uint64, error) { + val, closer, err := s.db.Get(appliedIndexKey) + + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + s.lastApplied = 0 + return 0, nil + } + return 0, err + } + + defer closer.Close() //nolint:errcheck + + s.lastApplied = binary.BigEndian.Uint64(val) + return s.lastApplied, nil +} + +// applyEntry applies a single Raft log entry to the batch and returns the result +// and any keys that were deleted (for DeleteBatchCmd). +// +// For StoreBatchCmd: the state machine delegates key generation to the shared +// EventRepository (same monotonic-ID counter used by standalone mode). The +// serialised StoredMessage bytes from the command buffer are passed directly to +// StoreWithBatch — no re-serialisation, no extra allocation. +func (s *EventStateMachine) applyEntry(batch storage.Batch, cmd []byte) (statemachine.Result, [][]byte) { + if len(cmd) == 0 { + return statemachine.Result{Value: 0}, nil + } + + switch CommandType(cmd[0]) { + case StoreBatchCmd: + items, err := UnmarshalStoreBatchCmd(cmd) + if err != nil { + log.Printf("raft: failed to unmarshal StoreBatchCmd: %v", err) + return statemachine.Result{Value: 0}, nil + } + for _, it := range items { + // StoreRawWithBatch takes the already-serialised value bytes and + // lets the repository assign the authoritative monotonic key. + // This is identical to the standalone write path — same ID counter, + // same key schema, no extra serialisation step. + if _, err := s.repo.StoreRawWithBatch(batch, it.Bucket, it.TopicHash, it.Indexes, it.Msg); err != nil { + log.Printf("raft: StoreRawWithBatch failed: %v", err) + return statemachine.Result{Value: 0}, nil + } + } + return statemachine.Result{Value: uint64(len(items))}, nil + + case DeleteBatchCmd: + keys, err := UnmarshalDeleteBatchCmd(cmd) + if err != nil { + log.Printf("raft: failed to unmarshal DeleteBatchCmd: %v", err) + return statemachine.Result{Value: 0}, nil + } + deleted := make([][]byte, 0, len(keys)) + for _, k := range keys { + kCopy := make([]byte, len(k)) + copy(kCopy, k) + if err := batch.Delete(kCopy); err != nil { + log.Printf("raft: batch.Delete failed: %v", err) + continue + } + deleted = append(deleted, kCopy) + } + return statemachine.Result{Value: uint64(len(deleted))}, deleted + + default: + log.Printf("raft: unknown command type: %d", cmd[0]) + return statemachine.Result{Value: 0}, nil + } +} + +func (s *EventStateMachine) Update(entries []statemachine.Entry) ([]statemachine.Entry, error) { + batch := s.db.NewBatch() + + defer batch.Close() //nolint:errcheck + + var allDeletedKeys [][]byte + + for i := range entries { + result, deletedKeys := s.applyEntry(batch, entries[i].Cmd) + entries[i].Result = result + if len(deletedKeys) > 0 { + allDeletedKeys = append(allDeletedKeys, deletedKeys...) + } + s.lastApplied = entries[i].Index + } + + idxBytes := make([]byte, 8) + binary.BigEndian.PutUint64(idxBytes, s.lastApplied) + if err := batch.Set(appliedIndexKey, idxBytes); err != nil { + return nil, err + } + + if err := batch.Commit(storage.NoSync); err != nil { + return nil, err + } + + if s.OnDeleteKeys != nil && len(allDeletedKeys) > 0 { + s.OnDeleteKeys(allDeletedKeys) + } + + return entries, nil +} + +func (s *EventStateMachine) Sync() error { + return s.db.Flush() +} + +func (s *EventStateMachine) Lookup(query interface{}) (interface{}, error) { + return nil, nil +} + +func (s *EventStateMachine) PrepareSnapshot() (interface{}, error) { + return s.lastApplied, nil +} + +func (s *EventStateMachine) SaveSnapshot(_ interface{}, w io.Writer, stopc <-chan struct{}) error { + return s.db.Scan(nil, func(key []byte, value []byte) error { + select { + case <-stopc: + return statemachine.ErrSnapshotStopped + default: + } + + // k := make([]byte, len(key)) + // v := make([]byte, len(value)) + // FIXME: no sure if I should copy the key/val or not + + if err := binary.Write(w, binary.LittleEndian, uint32(len(key))); err != nil { + return err + } + if _, err := w.Write(key); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, uint32(len(value))); err != nil { + return err + } + if _, err := w.Write(value); err != nil { + return err + } + + return nil + }) +} + +func (s *EventStateMachine) RecoverFromSnapshot(r io.Reader, stopc <-chan struct{}) error { + if err := s.clearDB(stopc); err != nil { + return err + } + + batch := s.db.NewBatch() + + defer batch.Close() //nolint:errcheck + + for { + select { + case <-stopc: + return statemachine.ErrSnapshotStopped + default: + } + + var klen uint32 + if err := binary.Read(r, binary.LittleEndian, &klen); err != nil { + if err == io.EOF { + break + } + return err + } + + k := make([]byte, klen) + if _, err := io.ReadFull(r, k); err != nil { + return err + } + + var vlen uint32 + if err := binary.Read(r, binary.LittleEndian, &vlen); err != nil { + return err + } + + v := make([]byte, vlen) + if _, err := io.ReadFull(r, v); err != nil { + return err + } + + if err := batch.Set(k, v); err != nil { + return err + } + } + + if err := batch.Commit(storage.Sync); err != nil { + return err + } + + val, closer, err := s.db.Get(appliedIndexKey) + if err == nil { + s.lastApplied = binary.BigEndian.Uint64(val) + defer closer.Close() //nolint:errcheck + } else if !errors.Is(err, pebble.ErrNotFound) { + return err + } + + return nil +} + +func (s *EventStateMachine) clearDB(stopc <-chan struct{}) error { + iter, err := s.db.NewIter(nil) + if err != nil { + return err + } + defer iter.Close() //nolint:errcheck + + batch := s.db.NewBatch() + + defer batch.Close() //nolint:errcheck + + for iter.First(); iter.Valid(); iter.Next() { + select { + case <-stopc: + return statemachine.ErrSnapshotStopped + default: + } + k := make([]byte, len(iter.Key())) + copy(k, iter.Key()) + if err := batch.Delete(k); err != nil { + return err + } + } + if err := iter.Error(); err != nil { + return err + } + + return batch.Commit(storage.NoSync) +} + +func (s *EventStateMachine) Close() error { + return s.db.Flush() +} diff --git a/internal/raft/event/statemachine_test.go b/internal/raft/event/statemachine_test.go new file mode 100644 index 0000000..6b9f611 --- /dev/null +++ b/internal/raft/event/statemachine_test.go @@ -0,0 +1,349 @@ +package raft + +import ( + "bytes" + "encoding/binary" + "testing" + "time" + + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/repository" + "github.com/futureq-io/futureq/internal/storage" + "github.com/lni/dragonboat/v4/statemachine" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" +) + +type EventStateMachineSuite struct { + suite.Suite + db storage.DB + repo *repository.EventRepository +} + +func TestEventStateMachineSuite(t *testing.T) { + suite.Run(t, new(EventStateMachineSuite)) +} + +func (s *EventStateMachineSuite) SetupTest() { + db, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + s.Require().NoError(err) + s.db = db + + repo, err := repository.NewEventRepository(db, zap.NewNop(), 1*time.Second) + s.Require().NoError(err) + s.repo = repo +} + +func (s *EventStateMachineSuite) TearDownTest() { + if s.db != nil { + s.db.Close() + } +} + +func (s *EventStateMachineSuite) newSM(onDelete func([][]byte)) *EventStateMachine { + factory := NewEventStateMachineFactory(s.db, s.repo, onDelete, zap.NewNop()) + sm, _ := factory(1, 1).(*EventStateMachine) + return sm +} + +// ─── Open ───────────────────────────────────────────────────────────────────── + +func (s *EventStateMachineSuite) TestOpen_FreshDB_ReturnsZero() { + require := s.Require() + + sm := s.newSM(nil) + + idx, err := sm.Open(nil) + require.NoError(err) + require.Equal(uint64(0), idx) +} + +func (s *EventStateMachineSuite) TestOpen_RestoresAppliedIndex() { + require := s.Require() + + // Seed applied index into DB. + b := s.db.NewBatch() + idxBytes := make([]byte, 8) + binary.BigEndian.PutUint64(idxBytes, 12345) + require.NoError(b.Set(appliedIndexKey, idxBytes)) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + sm := s.newSM(nil) + idx, err := sm.Open(nil) + require.NoError(err) + require.Equal(uint64(12345), idx) +} + +// ─── Update: StoreBatchCmd ──────────────────────────────────────────────────── + +func (s *EventStateMachineSuite) TestUpdate_StoreBatch_AppliesItems() { + require := s.Require() + + sm := s.newSM(nil) + + items := []StoreBatchItem{ + {Bucket: 1, TopicHash: 100, Msg: []byte("msg-a")}, + {Bucket: 2, TopicHash: 100, Msg: []byte("msg-b")}, + } + cmd, err := MarshalStoreBatchCmd(items) + require.NoError(err) + + entries := []statemachine.Entry{{Index: 1, Cmd: cmd}} + results, err := sm.Update(entries) + require.NoError(err) + require.Len(results, 1) + require.Equal(uint64(2), results[0].Result.Value, "should report 2 items applied") + + // Verify items are in storage via the repo's last-ID counter. + // Two items stored → repo's lastID should be 2. + // We verify indirectly by storing one more and checking its key's eventID. + b := s.db.NewBatch() + key, err := s.repo.StoreRawWithBatch(b, 1, 100, nil, []byte("third")) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + // The 3rd store should have eventID=3 (since 1,2 were used). + _, _, id, err := parseKeyEventID(key) + require.NoError(err) + require.Equal(uint64(3), id) +} + +// Helper to parse eventID from a key — small local helper to avoid circular import. +func parseKeyEventID(key []byte) (uint64, uint64, uint64, error) { + if len(key) != 24 { + return 0, 0, 0, bytes.ErrTooLarge + } + topicHash := binary.BigEndian.Uint64(key[0:8]) + bucket := binary.BigEndian.Uint64(key[8:16]) + eventID := binary.BigEndian.Uint64(key[16:24]) + return topicHash, bucket, eventID, nil +} + +func (s *EventStateMachineSuite) TestUpdate_EmptyCmd_Skipped() { + require := s.Require() + + sm := s.newSM(nil) + + entries := []statemachine.Entry{{Index: 1, Cmd: []byte{}}} + results, err := sm.Update(entries) + require.NoError(err) + require.Len(results, 1) + require.Equal(uint64(0), results[0].Result.Value) +} + +func (s *EventStateMachineSuite) TestUpdate_UnknownCmd_ReturnsZero() { + require := s.Require() + + sm := s.newSM(nil) + + entries := []statemachine.Entry{{Index: 1, Cmd: []byte{0xFF, 0x01}}} + results, err := sm.Update(entries) + require.NoError(err) + require.Equal(uint64(0), results[0].Result.Value) +} + +// ─── Update: DeleteBatchCmd ─────────────────────────────────────────────────── + +func (s *EventStateMachineSuite) TestUpdate_DeleteBatch_RemovesKeys() { + require := s.Require() + + sm := s.newSM(nil) + + // First store a message. + items := []StoreBatchItem{{Bucket: 1, TopicHash: 5, Msg: []byte("to-delete")}} + storeCmd, _ := MarshalStoreBatchCmd(items) + _, err := sm.Update([]statemachine.Entry{{Index: 1, Cmd: storeCmd}}) + require.NoError(err) + + // Find the stored key. + var storedKey []byte + err = s.db.Scan(nil, func(k, v []byte) error { + if len(k) == 24 { + storedKey = append([]byte(nil), k...) + } + return nil + }) + require.NoError(err) + require.NotNil(storedKey) + + // Now delete it. + deleteCmd, err := MarshalDeleteBatchCmd([][]byte{storedKey}) + require.NoError(err) + + results, err := sm.Update([]statemachine.Entry{{Index: 2, Cmd: deleteCmd}}) + require.NoError(err) + require.Equal(uint64(1), results[0].Result.Value, "should report 1 key deleted") + + // Verify it's gone. + _, _, err = s.db.Get(storedKey) + require.Error(err) +} + +func (s *EventStateMachineSuite) TestUpdate_DeleteBatch_CallsOnDeleteKeys() { + require := s.Require() + + var captured [][]byte + sm := s.newSM(func(keys [][]byte) { captured = keys }) + + key1 := make([]byte, 24) + key1[0] = 0x01 + key2 := make([]byte, 24) + key2[0] = 0x02 + + deleteCmd, _ := MarshalDeleteBatchCmd([][]byte{key1, key2}) + _, err := sm.Update([]statemachine.Entry{{Index: 1, Cmd: deleteCmd}}) + require.NoError(err) + + require.Len(captured, 2) +} + +func (s *EventStateMachineSuite) TestUpdate_NoDeleteCallback_DoesNotPanic() { + require := s.Require() + + sm := s.newSM(nil) // nil OnDeleteKeys + + key := make([]byte, 24) + deleteCmd, _ := MarshalDeleteBatchCmd([][]byte{key}) + _, err := sm.Update([]statemachine.Entry{{Index: 1, Cmd: deleteCmd}}) + require.NoError(err) // must not panic +} + +// ─── Update: lastApplied tracking ───────────────────────────────────────────── + +func (s *EventStateMachineSuite) TestUpdate_PersistsAppliedIndex() { + require := s.Require() + + sm := s.newSM(nil) + + items := []StoreBatchItem{{Bucket: 1, TopicHash: 1, Msg: []byte("m")}} + cmd, _ := MarshalStoreBatchCmd(items) + + _, err := sm.Update([]statemachine.Entry{{Index: 42, Cmd: cmd}}) + require.NoError(err) + + // Read the applied index back from storage. + val, closer, err := s.db.Get(appliedIndexKey) + require.NoError(err) + defer closer.Close() + require.Equal(uint64(42), binary.BigEndian.Uint64(val)) +} + +func (s *EventStateMachineSuite) TestUpdate_MultipleEntries_AdvancesAppliedIndex() { + require := s.Require() + + sm := s.newSM(nil) + + items := []StoreBatchItem{{Bucket: 1, TopicHash: 1, Msg: []byte("m")}} + cmd, _ := MarshalStoreBatchCmd(items) + + entries := []statemachine.Entry{ + {Index: 10, Cmd: cmd}, + {Index: 11, Cmd: cmd}, + {Index: 12, Cmd: cmd}, + } + _, err := sm.Update(entries) + require.NoError(err) + + val, closer, err := s.db.Get(appliedIndexKey) + require.NoError(err) + defer closer.Close() + require.Equal(uint64(12), binary.BigEndian.Uint64(val)) +} + +// ─── Sync / Lookup / PrepareSnapshot ───────────────────────────────────────── + +func (s *EventStateMachineSuite) TestSync_NoError() { + require := s.Require() + + sm := s.newSM(nil) + require.NoError(sm.Sync()) +} + +func (s *EventStateMachineSuite) TestLookup_ReturnsNil() { + require := s.Require() + + sm := s.newSM(nil) + result, err := sm.Lookup(nil) + require.NoError(err) + require.Nil(result) +} + +func (s *EventStateMachineSuite) TestPrepareSnapshot_ReturnsLastApplied() { + require := s.Require() + + sm := s.newSM(nil) + result, err := sm.PrepareSnapshot() + require.NoError(err) + require.Equal(uint64(0), result) // fresh SM +} + +// ─── Snapshot round-trip ────────────────────────────────────────────────────── + +// TestSnapshot_RoundTrip verifies that SaveSnapshot + RecoverFromSnapshot +// preserve both the data records and the applied-index metadata. +func (s *EventStateMachineSuite) TestSnapshot_RoundTrip() { + require := s.Require() + + sm := s.newSM(nil) + + // Seed data: applies one StoreBatchCmd at raft index 5. + items := []StoreBatchItem{ + {Bucket: 1, TopicHash: 7, Msg: []byte("snap-msg")}, + } + cmd, _ := MarshalStoreBatchCmd(items) + _, err := sm.Update([]statemachine.Entry{{Index: 5, Cmd: cmd}}) + require.NoError(err) + + // Save snapshot. + var buf bytes.Buffer + err = sm.SaveSnapshot(nil, &buf, nil) + require.NoError(err) + require.NotZero(buf.Len(), "snapshot must not be empty") + + appliedKeyBytes := []byte("metadata/raft/applied-index") + require.True(bytes.Contains(buf.Bytes(), appliedKeyBytes), + "snapshot must faithfully serialize the applied-index key") + + // Recover into a fresh DB. + db2, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + require.NoError(err) + defer db2.Close() + + repo2, err := repository.NewEventRepository(db2, zap.NewNop(), 1*time.Second) + require.NoError(err) + + factory2 := NewEventStateMachineFactory(db2, repo2, nil, zap.NewNop()) + sm2, _ := factory2(1, 1).(*EventStateMachine) + + err = sm2.RecoverFromSnapshot(&buf, nil) + require.NoError(err) + + // Applied index must be restored. + val, closer, err := db2.Get(appliedIndexKey) + require.NoError(err) + defer closer.Close() + require.Equal(uint64(5), binary.BigEndian.Uint64(val), + "recovered DB must contain the same applied index as was saved") + + // The stored message bytes must also be recoverable. + var found []byte + require.NoError(db2.Scan(nil, func(k, v []byte) error { + if len(k) == 24 { + found = append([]byte(nil), v...) + } + return nil + })) + require.Equal([]byte("snap-msg"), found, + "recovered DB must contain the same message bytes as was saved") +} + +// ─── Close ──────────────────────────────────────────────────────────────────── + +func (s *EventStateMachineSuite) TestClose_NoError() { + require := s.Require() + + sm := s.newSM(nil) + require.NoError(sm.Close()) +} diff --git a/internal/repository/events.go b/internal/repository/events.go new file mode 100644 index 0000000..ad9e490 --- /dev/null +++ b/internal/repository/events.go @@ -0,0 +1,208 @@ +package repository + +import ( + "encoding/binary" + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/proto" + "go.uber.org/zap" + + "github.com/futureq-io/futureq/internal/storage" + "github.com/futureq-io/futureq/pkg/utils" + storagepb "github.com/futureq-io/protocol/proto/go/storage" +) + +var eventsLastIDKey = []byte("metadata/event-repo/last-id") + +// EventRepository manages the monotonic event ID counter stored in Pebble. +type EventRepository struct { + db storage.DB + logger *zap.Logger + lastID uint64 + bucketSize time.Duration +} + +func NewEventRepository(db storage.DB, logger *zap.Logger, bucketSize time.Duration) (*EventRepository, error) { + repo := &EventRepository{ + db: db, + logger: logger, + bucketSize: bucketSize, + } + + val, closer, err := db.Get(eventsLastIDKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + repo.lastID = 0 + } else { + return nil, err + } + } else { + repo.lastID = binary.BigEndian.Uint64(val) + + defer closer.Close() //nolint:errcheck + } + + return repo, nil +} + +// StoreWithBatch marshals msg and adds it to an existing Pebble batch. +// It returns the generated 24-byte Pebble key for the caller to use as a delivery_tag. +func (er *EventRepository) StoreWithBatch(b storage.Batch, msg *storagepb.StoredMessage) ([]byte, error) { + nextID := atomic.AddUint64(&er.lastID, 1) + + fireAtMs := msg.EnqueuedAtUnixMs + msg.DelayMs + bucket := utils.CalculateBucket(fireAtMs, er.bucketSize) + topicHash := utils.TopicHash(msg.Topic) + key := utils.EventKey(bucket, topicHash, nextID) + + idBytes := make([]byte, 8) + binary.BigEndian.PutUint64(idBytes, nextID) + if err := b.Set(eventsLastIDKey, idBytes); err != nil { + return nil, err + } + + data, err := proto.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("failed to marshal message for topic %q: %w", msg.Topic, err) + } + + if err := b.Set(key, data); err != nil { + return nil, err + } + + for _, idx := range msg.GetIndexes() { + idxBytes, err := proto.Marshal(idx) + if err != nil { + return nil, fmt.Errorf("failed to marshal index to bytes: %w", err) + } + + if err := b.Set(idxBytes, key); err != nil { + return nil, err + } + } + + return key, nil +} + +// StoreWithBatch stores the raw msg value in bytes. +// This is used in Raft's write paths. +// It returns the generated 24-byte Pebble key for the caller to use as a delivery_tag. +func (er *EventRepository) StoreRawWithBatch(b storage.Batch, bucket uint64, topicHash uint64, indexes [][]byte, msg []byte) ([]byte, error) { + nextID := atomic.AddUint64(&er.lastID, 1) + + key := utils.EventKey(bucket, topicHash, nextID) + + idBytes := make([]byte, 8) + binary.BigEndian.PutUint64(idBytes, nextID) + if err := b.Set(eventsLastIDKey, idBytes); err != nil { + return nil, err + } + + if err := b.Set(key, msg); err != nil { + return nil, err + } + + for _, idx := range indexes { + if err := b.Set(idx, key); err != nil { + return nil, err + } + } + + return key, nil +} + +func (er *EventRepository) DeleteWithBatch(b storage.Batch, key []byte) error { + return b.Delete(key) +} + +type EventBatch struct { + b storage.Batch + repo *EventRepository +} + +func (er *EventRepository) NewBatch() *EventBatch { + return &EventBatch{ + b: er.db.NewBatch(), + repo: er, + } +} + +func (eb *EventBatch) Store(msg *storagepb.StoredMessage) ([]byte, error) { + nextID := atomic.AddUint64(&eb.repo.lastID, 1) + + fireAtMs := msg.EnqueuedAtUnixMs + msg.DelayMs + bucket := utils.CalculateBucket(fireAtMs, eb.repo.bucketSize) + topicHash := utils.TopicHash(msg.Topic) + key := utils.EventKey(bucket, topicHash, nextID) + + idBytes := make([]byte, 8) + binary.BigEndian.PutUint64(idBytes, nextID) + if err := eb.b.Set(eventsLastIDKey, idBytes); err != nil { + return nil, err + } + + data, err := proto.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("failed to marshal message for topic %q: %w", msg.Topic, err) + } + + if err := eb.b.Set(key, data); err != nil { + return nil, err + } + + for _, idx := range msg.GetIndexes() { + idxBytes, err := proto.Marshal(idx) + if err != nil { + return nil, fmt.Errorf("failed to marshal index to bytes: %w", err) + } + + if err := eb.b.Set(idxBytes, key); err != nil { + return nil, err + } + } + + return key, nil +} + +// StoreWithBatch stores the raw msg value in bytes. +// This is used in Raft's write paths. +// It returns the generated 24-byte Pebble key for the caller to use as a delivery_tag. +func (eb *EventBatch) StoreRaw(bucket uint64, topicHash uint64, indexes [][]byte, msg []byte) ([]byte, error) { + nextID := atomic.AddUint64(&eb.repo.lastID, 1) + + key := utils.EventKey(bucket, topicHash, nextID) + + idBytes := make([]byte, 8) + binary.BigEndian.PutUint64(idBytes, nextID) + if err := eb.b.Set(eventsLastIDKey, idBytes); err != nil { + return nil, err + } + + if err := eb.b.Set(key, msg); err != nil { + return nil, err + } + + for _, idx := range indexes { + if err := eb.b.Set(idx, key); err != nil { + return nil, err + } + } + + return key, nil +} + +func (eb *EventBatch) Delete(key []byte) error { + return eb.b.Delete(key) +} + +func (eb *EventBatch) Commit(mode storage.SyncMode) error { + return eb.b.Commit(mode) +} + +func (eb *EventBatch) Close() error { + return eb.b.Close() +} diff --git a/internal/repository/events_test.go b/internal/repository/events_test.go new file mode 100644 index 0000000..ea392d1 --- /dev/null +++ b/internal/repository/events_test.go @@ -0,0 +1,443 @@ +package repository + +import ( + "encoding/binary" + "sync" + "testing" + "time" + + "github.com/futureq-io/futureq/internal/config" + "github.com/futureq-io/futureq/internal/storage" + "github.com/futureq-io/futureq/pkg/utils" + pb "github.com/futureq-io/protocol/proto/go" + storagepb "github.com/futureq-io/protocol/proto/go/storage" + "google.golang.org/protobuf/proto" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" +) + +type EventRepositorySuite struct { + suite.Suite + db storage.DB + tmp string +} + +func TestEventRepositorySuite(t *testing.T) { + suite.Run(t, new(EventRepositorySuite)) +} + +func (s *EventRepositorySuite) SetupTest() { + db, err := storage.NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + s.Require().NoError(err) + s.db = db +} + +func (s *EventRepositorySuite) TearDownTest() { + if s.db != nil { + s.db.Close() + } +} + +func (s *EventRepositorySuite) newRepo(bucketSize time.Duration) *EventRepository { + repo, err := NewEventRepository(s.db, zap.NewNop(), bucketSize) + s.Require().NoError(err) + return repo +} + +// ─── Constructor ──────────────────────────────────────────────────────────── + +func (s *EventRepositorySuite) TestNewEventRepository_FreshDB_StartsAtZero() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + require.Equal(uint64(0), repo.lastID) +} + +func (s *EventRepositorySuite) TestNewEventRepository_RestoresLastID() { + require := s.Require() + + // Seed last-id directly into the DB. + b := s.db.NewBatch() + idBytes := make([]byte, 8) + binary.BigEndian.PutUint64(idBytes, 42) + require.NoError(b.Set(eventsLastIDKey, idBytes)) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + repo := s.newRepo(1 * time.Second) + require.Equal(uint64(42), repo.lastID) +} + +// ─── StoreWithBatch ───────────────────────────────────────────────────────── + +func (s *EventRepositorySuite) TestStoreWithBatch_AssignsMonotonicIDs() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + msg := &storagepb.StoredMessage{ + Topic: "orders", + Payload: []byte("hello"), + EnqueuedAtUnixMs: 17000, + DelayMs: 0, + } + + b := s.db.NewBatch() + key1, err := repo.StoreWithBatch(b, msg) + require.NoError(err) + key2, err := repo.StoreWithBatch(b, msg) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + // Parse IDs from keys — must be 1 and 2. + _, _, id1, err := utils.ParseEventKey(key1) + require.NoError(err) + _, _, id2, err := utils.ParseEventKey(key2) + require.NoError(err) + + require.Equal(uint64(1), id1) + require.Equal(uint64(2), id2) +} + +func (s *EventRepositorySuite) TestStoreWithBatch_KeyLayout() { + require := s.Require() + + bucketSize := 1 * time.Second + repo := s.newRepo(bucketSize) + + msg := &storagepb.StoredMessage{ + Topic: "payments", + Payload: []byte("data"), + EnqueuedAtUnixMs: 17000, // bucket 17 with 1s buckets + DelayMs: 2000, // fire at 19000 → bucket 19 + } + + b := s.db.NewBatch() + key, err := repo.StoreWithBatch(b, msg) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + th, bucket, _, err := utils.ParseEventKey(key) + require.NoError(err) + require.Equal(utils.TopicHash("payments"), th) + require.Equal(uint64(19), bucket) +} + +func (s *EventRepositorySuite) TestStoreWithBatch_StoredValueIsMarshalledMessage() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + msg := &storagepb.StoredMessage{ + Topic: "notifications", + Payload: []byte("payload-bytes"), + EnqueuedAtUnixMs: 20000, + DelayMs: 100, + TtlMs: 60000, + } + + b := s.db.NewBatch() + key, err := repo.StoreWithBatch(b, msg) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + val, closer, err := s.db.Get(key) + require.NoError(err) + defer closer.Close() + + var decoded storagepb.StoredMessage + require.NoError(proto.Unmarshal(val, &decoded)) + require.Equal(msg.Topic, decoded.Topic) + require.Equal(msg.Payload, decoded.Payload) + require.Equal(msg.EnqueuedAtUnixMs, decoded.EnqueuedAtUnixMs) + require.Equal(msg.DelayMs, decoded.DelayMs) + require.Equal(msg.TtlMs, decoded.TtlMs) +} + +func (s *EventRepositorySuite) TestStoreWithBatch_LastIDPersistedInBatch() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + b := s.db.NewBatch() + _, err := repo.StoreWithBatch(b, &storagepb.StoredMessage{ + Topic: "t", + EnqueuedAtUnixMs: 1000, + }) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + val, closer, err := s.db.Get(eventsLastIDKey) + require.NoError(err) + defer closer.Close() + + require.Equal(uint64(1), binary.BigEndian.Uint64(val)) +} + +func (s *EventRepositorySuite) TestStoreWithBatch_IndexesAreStored() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + msg := &storagepb.StoredMessage{ + Topic: "indexed-topic", + EnqueuedAtUnixMs: 1000, + Indexes: []*pb.Index{}, + } + + b := s.db.NewBatch() + key, err := repo.StoreWithBatch(b, msg) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + // With no indexes, the only new key is the event key itself. + th, bucket, id, err := utils.ParseEventKey(key) + require.NoError(err) + require.Equal(utils.TopicHash("indexed-topic"), th) + require.Equal(uint64(1), bucket) // 1000ms / 1s + require.Equal(uint64(1), id) +} + +// ─── StoreRawWithBatch ────────────────────────────────────────────────────── + +func (s *EventRepositorySuite) TestStoreRawWithBatch_StoresRawBytes() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + rawMsg := []byte("raw-protobuf-bytes") + indexes := [][]byte{[]byte("idx-key-1")} + + b := s.db.NewBatch() + key, err := repo.StoreRawWithBatch(b, 42, 12345, indexes, rawMsg) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + // Key layout must match supplied bucket and topicHash. + th, bucket, id, err := utils.ParseEventKey(key) + require.NoError(err) + require.Equal(uint64(12345), th) + require.Equal(uint64(42), bucket) + require.Equal(uint64(1), id) + + // Value must be the exact raw bytes — no re-serialisation. + val, closer, err := s.db.Get(key) + require.NoError(err) + defer closer.Close() + require.Equal(rawMsg, val) + + // Index key must map back to event key. + idxVal, idxCloser, err := s.db.Get([]byte("idx-key-1")) + require.NoError(err) + defer idxCloser.Close() + require.Equal(key, idxVal) +} + +func (s *EventRepositorySuite) TestStoreRawWithBatch_MultipleIndexes() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + indexes := [][]byte{ + []byte("idx-a"), + []byte("idx-b"), + []byte("idx-c"), + } + + b := s.db.NewBatch() + key, err := repo.StoreRawWithBatch(b, 1, 2, indexes, []byte("v")) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + for _, idx := range indexes { + val, closer, err := s.db.Get(idx) + require.NoError(err) + require.Equal(key, val) + closer.Close() + } +} + +func (s *EventRepositorySuite) TestStoreRawWithBatch_UpdatesLastID() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + b := s.db.NewBatch() + _, err := repo.StoreRawWithBatch(b, 1, 1, nil, []byte("x")) + require.NoError(err) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + val, closer, err := s.db.Get(eventsLastIDKey) + require.NoError(err) + defer closer.Close() + require.Equal(uint64(1), binary.BigEndian.Uint64(val)) +} + +// ─── DeleteWithBatch ──────────────────────────────────────────────────────── + +func (s *EventRepositorySuite) TestDeleteWithBatch_RemovesKey() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + // Store first. + b1 := s.db.NewBatch() + key, err := repo.StoreWithBatch(b1, &storagepb.StoredMessage{ + Topic: "del-topic", + EnqueuedAtUnixMs: 5000, + }) + require.NoError(err) + require.NoError(b1.Commit(storage.Sync)) + require.NoError(b1.Close()) + + // Now delete it. + b2 := s.db.NewBatch() + require.NoError(repo.DeleteWithBatch(b2, key)) + require.NoError(b2.Commit(storage.Sync)) + require.NoError(b2.Close()) + + _, _, err = s.db.Get(key) + require.Error(err, "expected key to be deleted") +} + +// ─── EventBatch ───────────────────────────────────────────────────────────── + +func (s *EventRepositorySuite) TestEventBatch_Store_And_Commit() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + eb := repo.NewBatch() + key, err := eb.Store(&storagepb.StoredMessage{ + Topic: "batch-topic", + Payload: []byte("batched"), + EnqueuedAtUnixMs: 9000, + }) + require.NoError(err) + require.NoError(eb.Commit(storage.Sync)) + require.NoError(eb.Close()) + + val, closer, err := s.db.Get(key) + require.NoError(err) + defer closer.Close() + + var decoded storagepb.StoredMessage + require.NoError(proto.Unmarshal(val, &decoded)) + require.Equal("batch-topic", decoded.Topic) +} + +func (s *EventRepositorySuite) TestEventBatch_Store_IsMonotonicWithinBatch() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + eb := repo.NewBatch() + key1, err := eb.Store(&storagepb.StoredMessage{Topic: "t", EnqueuedAtUnixMs: 1000}) + require.NoError(err) + key2, err := eb.Store(&storagepb.StoredMessage{Topic: "t", EnqueuedAtUnixMs: 1000}) + require.NoError(err) + require.NoError(eb.Commit(storage.Sync)) + require.NoError(eb.Close()) + + _, _, id1, _ := utils.ParseEventKey(key1) + _, _, id2, _ := utils.ParseEventKey(key2) + require.Less(id1, id2) +} + +func (s *EventRepositorySuite) TestEventBatch_StoreRaw() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + eb := repo.NewBatch() + key, err := eb.StoreRaw(7, 99, nil, []byte("raw")) + require.NoError(err) + require.NoError(eb.Commit(storage.Sync)) + require.NoError(eb.Close()) + + th, bucket, _, err := utils.ParseEventKey(key) + require.NoError(err) + require.Equal(uint64(99), th) + require.Equal(uint64(7), bucket) +} + +func (s *EventRepositorySuite) TestEventBatch_Delete() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + // Seed a key directly. + b := s.db.NewBatch() + require.NoError(b.Set([]byte("to-delete"), []byte("v"))) + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + + eb := repo.NewBatch() + require.NoError(eb.Delete([]byte("to-delete"))) + require.NoError(eb.Commit(storage.Sync)) + require.NoError(eb.Close()) + + _, _, err := s.db.Get([]byte("to-delete")) + require.Error(err) +} + +// ─── Concurrency ──────────────────────────────────────────────────────────── + +func (s *EventRepositorySuite) TestStoreWithBatch_Concurrent_IDsAreUnique() { + require := s.Require() + + repo := s.newRepo(1 * time.Second) + + const goroutines = 10 + const perGoroutine = 20 + + batches := make([]storage.Batch, goroutines) + for i := range batches { + batches[i] = s.db.NewBatch() + } + + var wg sync.WaitGroup + keys := make([][]byte, goroutines*perGoroutine) + errs := make([]error, goroutines*perGoroutine) + + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < perGoroutine; i++ { + k, err := repo.StoreWithBatch(batches[g], &storagepb.StoredMessage{ + Topic: "concurrent", + EnqueuedAtUnixMs: 1000, + }) + keys[g*perGoroutine+i] = k + errs[g*perGoroutine+i] = err + } + }(g) + } + wg.Wait() + + for _, err := range errs { + require.NoError(err) + } + + // Commit all batches. + for _, b := range batches { + require.NoError(b.Commit(storage.Sync)) + require.NoError(b.Close()) + } + + // All event IDs must be unique. + seen := make(map[uint64]struct{}) + for _, k := range keys { + _, _, id, err := utils.ParseEventKey(k) + require.NoError(err) + _, exists := seen[id] + require.False(exists, "duplicate event ID %d", id) + seen[id] = struct{}{} + } + + require.Len(seen, goroutines*perGoroutine) +} diff --git a/internal/storage/bbolt.go b/internal/storage/bbolt.go new file mode 100644 index 0000000..dba7d4e --- /dev/null +++ b/internal/storage/bbolt.go @@ -0,0 +1,292 @@ +package storage + +import ( + "bytes" + "fmt" + "io" + + "github.com/futureq-io/futureq/internal/config" + bolt "go.etcd.io/bbolt" +) + +// ── bbolt adapter ───────────────────────────────────────────────────────────── +// +// bbolt stores data inside named buckets; we use a single bucket to preserve +// the flat, lexicographically ordered keyspace the dispatcher and key schema +// depend on. +// +// Concurrency model: +// - Reads: each NewIter call opens a read-only transaction. The transaction +// is committed (released) when the iterator is closed. +// - Writes: each Batch.Commit call opens and commits a read-write transaction. + +const defaultBucket = "futureq" + +// BoltConfig holds the configuration needed to open a bbolt database. +type BoltConfig struct { + // DataPath is the path to the bbolt database file (e.g. "/data/futureq.db"). + DataPath string + // Bucket is the name of the bbolt bucket used to store all records. + // Defaults to "futureq" if empty. + Bucket string +} + +// boltDB wraps *bolt.DB and implements storage.DB. +type boltDB struct { + db *bolt.DB + bucket []byte +} + +// NewBoltDB opens a bbolt database at cfg.DataPath and returns it as a +// storage.DB. The database file is created if it does not exist. +func NewBoltDB(cfg config.Bolt) (DB, error) { + db, err := bolt.Open(cfg.DataPath, 0600, nil) + if err != nil { + return nil, fmt.Errorf("bbolt: failed to open %q: %w", cfg.DataPath, err) + } + + bname := cfg.DefaultBucket + if bname == "" { + bname = defaultBucket + } + + // Ensure the bucket exists before any caller tries to use it. + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bname)) + return err + }); err != nil { + _ = db.Close() + return nil, fmt.Errorf("bbolt: failed to create bucket %q: %w", bname, err) + } + + return &boltDB{db: db, bucket: []byte(bname)}, nil +} + +// Get retrieves the value for key using a short-lived read-only transaction. +func (b *boltDB) Get(key []byte) ([]byte, io.Closer, error) { + var out []byte + err := b.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(b.bucket) + if bkt == nil { + return fmt.Errorf("bbolt: bucket %q not found", b.bucket) + } + v := bkt.Get(key) + if v == nil { + return ErrNotFound + } + // Values are only valid for the duration of the transaction, so copy. + out = make([]byte, len(v)) + copy(out, v) + return nil + }) + return out, io.NopCloser(nil), err +} + +// NewBatch returns a boltBatch that accumulates Set/Delete operations and +// applies them in a single read-write transaction on Commit. +func (b *boltDB) NewBatch() Batch { + return &boltBatch{db: b.db, bucket: b.bucket} +} + +// NewIter opens a read-only transaction and returns an iterator over the bucket. +// The transaction is released when the iterator is closed. +func (b *boltDB) NewIter(opts *IterOptions) (Iterator, error) { + tx, err := b.db.Begin(false) + if err != nil { + return nil, fmt.Errorf("bbolt: failed to begin read tx: %w", err) + } + + bkt := tx.Bucket(b.bucket) + if bkt == nil { + _ = tx.Rollback() + return nil, fmt.Errorf("bbolt: bucket %q not found", b.bucket) + } + + var lower, upper []byte + if opts != nil { + lower = opts.LowerBound + upper = opts.UpperBound + } + + return &boltIterator{ + tx: tx, + cursor: bkt.Cursor(), + lower: lower, + upper: upper, + }, nil +} + +// Scan iterates over the bbolt database and yields keys and values to the provided function. +func (b *boltDB) Scan(opts *IterOptions, yield func(key, value []byte) error) error { + // Open a read-only transaction. + // This provides the same consistency guarantees as Pebble's Snapshot. + return b.db.View(func(tx *bolt.Tx) error { + // bbolt stores data in buckets. Grab the default bucket for your KV store. + bucket := tx.Bucket(b.bucket) + if bucket == nil { + // If the bucket doesn't exist, the database is effectively empty. + return nil + } + + c := bucket.Cursor() + var k, v []byte + + // 1. Handle LowerBound + if opts != nil && opts.LowerBound != nil { + // Seek moves the cursor to the first key that is >= LowerBound + k, v = c.Seek(opts.LowerBound) + } else { + // No LowerBound? Start at the very beginning of the database + k, v = c.First() + } + + // 2. The hot loop + for k != nil { + // Handle UpperBound (LevelDB/Pebble standard is that UpperBound is exclusive) + // bytes.Compare returns >= 0 if k is equal to or greater than UpperBound + if opts != nil && opts.UpperBound != nil && bytes.Compare(k, opts.UpperBound) >= 0 { + break + } + + // Yield to the caller. If they return false, break the loop early. + if err := yield(k, v); err != nil { + return fmt.Errorf("yield func: %w", err) + } + + // Move to the next key in the B+Tree + k, v = c.Next() + } + + return nil + }) +} + +// Flush is a no-op for bbolt: every committed transaction is fsync'd by default. +func (b *boltDB) Flush() error { return nil } + +// Close closes the underlying bbolt database. +func (b *boltDB) Close() error { return b.db.Close() } + +// ── boltBatch ──────────────────────────────────────────────────────────────── + +type boltBatch struct { + db *bolt.DB + bucket []byte + ops []boltOp +} + +type boltOp struct { + del bool + key []byte + value []byte +} + +func (bb *boltBatch) Set(key, value []byte) error { + k := make([]byte, len(key)) + copy(k, key) + v := make([]byte, len(value)) + copy(v, value) + bb.ops = append(bb.ops, boltOp{key: k, value: v}) + return nil +} + +func (bb *boltBatch) Delete(key []byte) error { + k := make([]byte, len(key)) + copy(k, key) + bb.ops = append(bb.ops, boltOp{del: true, key: k}) + return nil +} + +// Commit applies all accumulated operations in a single read-write transaction. +// SyncMode is accepted for interface compatibility; bbolt always syncs on +// commit unless bolt.DB.NoSync is set globally. +func (bb *boltBatch) Commit(_ SyncMode) error { + return bb.db.Update(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bb.bucket) + if bkt == nil { + return fmt.Errorf("bbolt: bucket %q not found", bb.bucket) + } + for _, op := range bb.ops { + var err error + if op.del { + err = bkt.Delete(op.key) + } else { + err = bkt.Put(op.key, op.value) + } + if err != nil { + return err + } + } + return nil + }) +} + +// Close discards the pending operations. Safe to call after Commit. +func (bb *boltBatch) Close() error { + bb.ops = bb.ops[:0] + return nil +} + +// ── boltIterator ───────────────────────────────────────────────────────────── + +// boltIterator adapts a *bolt.Cursor to the storage.Iterator interface. +// It owns the read-only transaction it was created from; Close() rolls it back. +type boltIterator struct { + tx *bolt.Tx + cursor *bolt.Cursor + lower []byte + upper []byte + + curKey []byte + curVal []byte + done bool +} + +func (bi *boltIterator) First() bool { + var k, v []byte + if bi.lower != nil { + k, v = bi.cursor.Seek(bi.lower) + } else { + k, v = bi.cursor.First() + } + return bi.set(k, v) +} + +func (bi *boltIterator) Next() bool { + if bi.done { + return false + } + k, v := bi.cursor.Next() + return bi.set(k, v) +} + +func (bi *boltIterator) set(k, v []byte) bool { + if k == nil || (bi.upper != nil && bytes.Compare(k, bi.upper) >= 0) { + bi.done = true + bi.curKey = nil + bi.curVal = nil + return false + } + bi.curKey = k + bi.curVal = v + return true +} + +func (bi *boltIterator) Valid() bool { return !bi.done && bi.curKey != nil } +func (bi *boltIterator) Key() []byte { return bi.curKey } +func (bi *boltIterator) Value() []byte { return bi.curVal } + +// Error always returns nil for bbolt — range exhaustion surfaces as nil keys +// from Next()/First(), not as deferred error values. +func (bi *boltIterator) Error() error { return nil } + +// Close rolls back the read-only transaction, releasing it back to bbolt. +func (bi *boltIterator) Close() error { return bi.tx.Rollback() } + +// ── compile-time interface checks ───────────────────────────────────────────── + +var ( + _ DB = (*boltDB)(nil) + _ Batch = (*boltBatch)(nil) + _ Iterator = (*boltIterator)(nil) +) diff --git a/internal/storage/contract.go b/internal/storage/contract.go new file mode 100644 index 0000000..d1fcc5e --- /dev/null +++ b/internal/storage/contract.go @@ -0,0 +1,130 @@ +package storage + +import ( + "errors" + "io" +) + +// ErrNotFound is returned by DB.Get when the requested key does not exist. +// Implementations must map their own not-found sentinel to this error so that +// callers don't need to import engine-specific packages. +var ErrNotFound = errors.New("key not found") + +// SyncMode controls fsync behaviour on a Batch commit. +type SyncMode uint8 + +const ( + // NoSync commits without an fsync. Faster, but data may be lost on crash. + NoSync SyncMode = iota + // Sync commits with an fsync. Slower, durable on power loss. + Sync +) + +// IterOptions configures an Iterator's key range. +// Both bounds are optional; a nil bound means the iterator is unbounded in +// that direction. +type IterOptions struct { + // LowerBound is the inclusive lower bound of the iteration range. + // The iterator will not visit keys less than this value. + LowerBound []byte + + // UpperBound is the exclusive upper bound of the iteration range. + // The iterator will not visit keys greater than or equal to this value. + UpperBound []byte +} + +// Iterator is a forward cursor over a sorted key-value range. +// Callers must call Close when done to release underlying resources. +// +// The byte slices returned by Key and Value are only valid until the next +// call to any iterator method or until Close is called. Copy them if you +// need them to outlive the current iteration step. +type Iterator interface { + // First positions the iterator at the first key (respecting LowerBound). + // Returns true if a key is found. + First() bool + + // Next advances the iterator to the next key. + // Returns true if a key is found. + Next() bool + + // Valid reports whether the iterator is positioned at a valid key. + Valid() bool + + // Key returns the key at the current position. + // The slice is only valid until the next iterator method call. + Key() []byte + + // Value returns the value at the current position. + // The slice is only valid until the next iterator method call. + Value() []byte + + // Error returns any accumulated error from iteration. + // Always check this after the loop exits. + Error() error + + // Close releases the iterator's resources. + Close() error +} + +// Batch is a collection of mutations (Set and Delete) that are applied to the +// database atomically on Commit. Batches are not safe for concurrent use. +// +// Callers must call Close when done, regardless of whether Commit was called, +// to release pooled resources. Closing after a successful Commit is a no-op. +type Batch interface { + // Set adds a key-value pair to the batch, overwriting any existing value. + Set(key, value []byte) error + + // Delete removes a key from the batch. + Delete(key []byte) error + + // Commit applies all mutations in the batch to the database atomically. + // mode controls whether the commit is durably synced to disk. + Commit(mode SyncMode) error + + // Close releases the batch's resources. Safe to call after Commit. + Close() error +} + +// DB is the engine-agnostic storage interface. +// +// Implementations are free to be backed by Pebble, BoltDB, or any other +// ordered key-value engine. +// +// Key ordering guarantee: implementations MUST maintain keys in +// lexicographic (byte-wise) order so that the dispatcher's range scan and the +// bucket-based key scheme work correctly. +type DB interface { + // Get retrieves the value stored for key. + // Returns ErrNotFound if the key does not exist. + Get(key []byte) (value []byte, closer io.Closer, err error) + + // NewBatch returns a new empty Batch. + // The caller must call Batch.Close when done. + NewBatch() Batch + + // NewIter returns a consistent, point-in-time iterator over the database. + // opts may be nil for an unbounded scan. + // The caller must call Iterator.Close when done; this also releases any + // underlying transaction or snapshot held by the iterator. + NewIter(opts *IterOptions) (Iterator, error) + + // Scan iterates over the database and calls yield for each key-value pair. + // + // MEMORY WARNING: The key and value slices passed to yield are owned by the + // underlying storage engine (e.g., mmap for Bolt, block cache for Pebble). + // They are ONLY valid for the duration of the yield function call. + // + // You MUST NOT retain references to key or value after yield returns, + // and you MUST NOT modify the bytes within the slices. If you need to keep + // the data, you must copy it (e.g., append([]byte(nil), key...)). + Scan(opts *IterOptions, yield func(key, value []byte) error) error + + // Flush forces any in-memory data to be written to durable storage. + Flush() error + + // Close shuts down the storage engine, flushing all pending writes. + // No other methods may be called after Close returns. + Close() error +} diff --git a/internal/storage/contract_test.go b/internal/storage/contract_test.go new file mode 100644 index 0000000..40e2c96 --- /dev/null +++ b/internal/storage/contract_test.go @@ -0,0 +1,512 @@ +package storage + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/suite" +) + +// EngineFactory creates a fresh, isolated storage.DB for each test invocation. +// Implementations must return an empty database and register any cleanup +// (file removal, etc.) via the provided *testing.T. +type EngineFactory func(t *testing.T) DB + +// DBContractSuite verifies that any storage.DB implementation honours the +// semantics defined in contract.go: lexicographic ordering, batch atomicity, +// exclusive upper bounds, iterator/scan equivalence, and the ErrNotFound +// sentinel. Run it once per engine. +type DBContractSuite struct { + suite.Suite + NewEngine EngineFactory + db DB +} + +// TestDBContract is the entry point used by per-engine test files: +// +// func TestPebbleContract(t *testing.T) { +// suite.Run(t, &DBContractSuite{NewEngine: newPebbleForTest}) +// } +func TestDBContract(t *testing.T) { + // Placeholder so this file is a valid test target; real suites are + // registered by the per-engine files below. +} + +func (s *DBContractSuite) SetupTest() { + s.Require().NotNil(s.NewEngine, "DBContractSuite.NewEngine must be set") + s.db = s.NewEngine(s.T()) +} + +func (s *DBContractSuite) TearDownTest() { + if s.db != nil { + s.db.Close() + } +} + +// ─── Get / Set round-trip ─────────────────────────────────────────────────── + +func (s *DBContractSuite) TestGet_Set_RoundTrip() { + require := s.Require() + + b := s.db.NewBatch() + require.NoError(b.Set([]byte("hello"), []byte("world"))) + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) + + val, closer, err := s.db.Get([]byte("hello")) + require.NoError(err) + require.Equal([]byte("world"), val) + require.NoError(closer.Close()) +} + +// DRIFT NOTE: the contract in storage/contract.go says "Implementations must +// map their own not-found sentinel to this error" (ErrNotFound). Bolt does +// map bbolt's nil-return to ErrNotFound; Pebble does NOT — it returns +// pebble.ErrNotFound directly. This is drift between the two engines: the +// callers of storage.DB cannot rely on errors.Is(err, storage.ErrNotFound) +// unless Pebble's Get is patched. The test below enforces only that *some* +// error is returned; engines are free to return either sentinel. +func (s *DBContractSuite) TestGet_NotFound_ReturnsError() { + require := s.Require() + + _, _, err := s.db.Get([]byte("definitely-missing-key")) + require.Error(err, "Get on missing key must return a non-nil error") +} + +func (s *DBContractSuite) TestGet_ReturnsCopy_NotAlias() { + require := s.Require() + + original := []byte("value") + b := s.db.NewBatch() + require.NoError(b.Set([]byte("k"), original)) + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) + + // Mutating the source slice must not affect the stored value. + original[0] = 'X' + + val, closer, err := s.db.Get([]byte("k")) + require.NoError(err) + defer closer.Close() + require.Equal([]byte("value"), val, + "stored value must be independent of the caller's slice") +} + +func (s *DBContractSuite) TestGet_EmptyValue() { + require := s.Require() + + b := s.db.NewBatch() + require.NoError(b.Set([]byte("empty"), []byte{})) + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) + + val, closer, err := s.db.Get([]byte("empty")) + require.NoError(err) + defer closer.Close() + require.Empty(val) +} + +// ─── Batch ────────────────────────────────────────────────────────────────── + +func (s *DBContractSuite) TestBatch_SetDelete_AtomicCommit() { + require := s.Require() + + // Seed both keys. + b1 := s.db.NewBatch() + require.NoError(b1.Set([]byte("a"), []byte("1"))) + require.NoError(b1.Set([]byte("b"), []byte("2"))) + require.NoError(b1.Commit(Sync)) + require.NoError(b1.Close()) + + // Delete one, overwrite the other, in a single batch. + b2 := s.db.NewBatch() + require.NoError(b2.Delete([]byte("a"))) + require.NoError(b2.Set([]byte("b"), []byte("updated"))) + require.NoError(b2.Commit(Sync)) + require.NoError(b2.Close()) + + // Deleted key must not be retrievable. + _, _, err := s.db.Get([]byte("a")) + require.Error(err) + + val, closer, err := s.db.Get([]byte("b")) + require.NoError(err) + defer closer.Close() + require.Equal([]byte("updated"), val) +} + +func (s *DBContractSuite) TestBatch_DeleteNonExistent_NoError() { + require := s.Require() + + // Deleting a key that does not exist must be a silent no-op. + b := s.db.NewBatch() + require.NoError(b.Delete([]byte("never-existed"))) + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) +} + +func (s *DBContractSuite) TestBatch_CommitNoSync_Succeeds() { + require := s.Require() + + b := s.db.NewBatch() + require.NoError(b.Set([]byte("k"), []byte("v"))) + require.NoError(b.Commit(NoSync)) + require.NoError(b.Close()) + + val, closer, err := s.db.Get([]byte("k")) + require.NoError(err) + defer closer.Close() + require.Equal([]byte("v"), val) +} + +func (s *DBContractSuite) TestBatch_CloseAfterCommit_NoError() { + require := s.Require() + + b := s.db.NewBatch() + require.NoError(b.Set([]byte("k"), []byte("v"))) + require.NoError(b.Commit(Sync)) + // Close after Commit must be a safe no-op per the Batch contract. + require.NoError(b.Close()) +} + +func (s *DBContractSuite) TestBatch_OverwriteWithinBatch_LastWriteWins() { + require := s.Require() + + b := s.db.NewBatch() + require.NoError(b.Set([]byte("k"), []byte("first"))) + require.NoError(b.Set([]byte("k"), []byte("second"))) + require.NoError(b.Set([]byte("k"), []byte("third"))) + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) + + val, closer, err := s.db.Get([]byte("k")) + require.NoError(err) + defer closer.Close() + require.Equal([]byte("third"), val) +} + +// ─── Iterator ─────────────────────────────────────────────────────────────── + +func (s *DBContractSuite) seed(keys ...string) { + b := s.db.NewBatch() + for _, k := range keys { + s.Require().NoError(b.Set([]byte(k), []byte("val-"+k))) + } + s.Require().NoError(b.Commit(Sync)) + s.Require().NoError(b.Close()) +} + +func (s *DBContractSuite) TestIterator_LexicographicOrder() { + require := s.Require() + + s.seed("delta", "alpha", "charlie", "bravo") + + iter, err := s.db.NewIter(nil) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + var got []string + for iter.First(); iter.Valid(); iter.Next() { + got = append(got, string(iter.Key())) + } + require.NoError(iter.Error()) + require.Equal([]string{"alpha", "bravo", "charlie", "delta"}, got, + "engines MUST maintain lexicographic (byte-wise) key order") +} + +func (s *DBContractSuite) TestIterator_BinaryKeys_ByteOrder() { + require := s.Require() + + // Keys with high-bit bytes — exercises raw byte ordering, not string collation. + b := s.db.NewBatch() + keys := [][]byte{ + {0xFF}, + {0x00}, + {0x80}, + {0x01}, + } + for _, k := range keys { + require.NoError(b.Set(k, []byte("v"))) + } + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) + + iter, err := s.db.NewIter(nil) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + var got [][]byte + for iter.First(); iter.Valid(); iter.Next() { + k := append([]byte(nil), iter.Key()...) + got = append(got, k) + } + require.Equal([][]byte{{0x00}, {0x01}, {0x80}, {0xFF}}, got) +} + +func (s *DBContractSuite) TestIterator_LowerBound_Inclusive() { + require := s.Require() + + s.seed("a", "b", "c", "d") + + iter, err := s.db.NewIter(&IterOptions{LowerBound: []byte("b")}) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + var got []string + for iter.First(); iter.Valid(); iter.Next() { + got = append(got, string(iter.Key())) + } + require.Equal([]string{"b", "c", "d"}, got, + "LowerBound is inclusive: key 'b' must be visited") +} + +func (s *DBContractSuite) TestIterator_UpperBound_Exclusive() { + require := s.Require() + + s.seed("a", "b", "c", "d") + + iter, err := s.db.NewIter(&IterOptions{UpperBound: []byte("c")}) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + var got []string + for iter.First(); iter.Valid(); iter.Next() { + got = append(got, string(iter.Key())) + } + require.Equal([]string{"a", "b"}, got, + "UpperBound is exclusive: key 'c' must NOT be visited") +} + +func (s *DBContractSuite) TestIterator_LowerAndUpperBound_Range() { + require := s.Require() + + s.seed("a", "b", "c", "d", "e") + + iter, err := s.db.NewIter(&IterOptions{ + LowerBound: []byte("b"), + UpperBound: []byte("d"), + }) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + var got []string + for iter.First(); iter.Valid(); iter.Next() { + got = append(got, string(iter.Key())) + } + require.Equal([]string{"b", "c"}, got) +} + +func (s *DBContractSuite) TestIterator_EmptyRange_YieldsNothing() { + require := s.Require() + + s.seed("a", "b", "c") + + // LowerBound beyond all keys. + iter, err := s.db.NewIter(&IterOptions{LowerBound: []byte("zzz")}) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + require.False(iter.First(), "empty range: First() must return false") + require.False(iter.Valid()) +} + +func (s *DBContractSuite) TestIterator_EmptyDB_YieldsNothing() { + require := s.Require() + + iter, err := s.db.NewIter(nil) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + require.False(iter.First()) + require.False(iter.Valid()) + require.NoError(iter.Error()) +} + +func (s *DBContractSuite) TestIterator_KeyValue_Pairs() { + require := s.Require() + + s.seed("only") + + iter, err := s.db.NewIter(nil) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + require.True(iter.First()) + require.Equal([]byte("only"), iter.Key()) + require.Equal([]byte("val-only"), iter.Value()) + require.False(iter.Next()) + require.False(iter.Valid()) +} + +func (s *DBContractSuite) TestIterator_NilOptions_UnboundedScan() { + require := s.Require() + + s.seed("x", "y", "z") + + iter, err := s.db.NewIter(nil) + require.NoError(err) + defer iter.Close() //nolint:errcheck + + count := 0 + for iter.First(); iter.Valid(); iter.Next() { + count++ + } + require.Equal(3, count, "nil IterOptions must scan everything") +} + +// ─── Scan ─────────────────────────────────────────────────────────────────── + +func (s *DBContractSuite) TestScan_VisitsAllKeysInOrder() { + require := s.Require() + + s.seed("m1", "m2", "m3", "m4") + + var visited []string + err := s.db.Scan(nil, func(k, v []byte) error { + visited = append(visited, string(k)) + return nil + }) + require.NoError(err) + require.Equal([]string{"m1", "m2", "m3", "m4"}, visited) +} + +func (s *DBContractSuite) TestScan_WithBounds_RespectsThem() { + require := s.Require() + + s.seed("a", "b", "c", "d") + + var visited []string + err := s.db.Scan(&IterOptions{ + LowerBound: []byte("b"), + UpperBound: []byte("d"), + }, func(k, v []byte) error { + visited = append(visited, string(k)) + return nil + }) + require.NoError(err) + require.Equal([]string{"b", "c"}, visited) +} + +func (s *DBContractSuite) TestScan_YieldError_StopsAndPropagates() { + require := s.Require() + + s.seed("k1", "k2", "k3") + + sentinel := fmt.Errorf("stop here") + count := 0 + err := s.db.Scan(nil, func(k, v []byte) error { + count++ + return sentinel + }) + require.Error(err, "yield error must propagate to the caller") + require.Equal(1, count, "scan must stop at the first yield error") +} + +func (s *DBContractSuite) TestScan_EmptyDB_CallsYieldZeroTimes() { + require := s.Require() + + called := false + err := s.db.Scan(nil, func(k, v []byte) error { + called = true + return nil + }) + require.NoError(err) + require.False(called) +} + +func (s *DBContractSuite) TestScan_ValuesMatchStored() { + require := s.Require() + + b := s.db.NewBatch() + require.NoError(b.Set([]byte("k"), []byte("expected-value"))) + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) + + var gotVal []byte + err := s.db.Scan(nil, func(k, v []byte) error { + gotVal = append([]byte(nil), v...) + return nil + }) + require.NoError(err) + require.Equal([]byte("expected-value"), gotVal) +} + +// ─── Consistency: iterator vs scan ────────────────────────────────────────── + +func (s *DBContractSuite) TestIteratorAndScan_SeeSameData() { + require := s.Require() + + s.seed("p", "q", "r") + + var viaIter []string + iter, err := s.db.NewIter(nil) + require.NoError(err) + for iter.First(); iter.Valid(); iter.Next() { + viaIter = append(viaIter, string(iter.Key())) + } + require.NoError(iter.Close()) + + var viaScan []string + require.NoError(s.db.Scan(nil, func(k, v []byte) error { + viaScan = append(viaScan, string(k)) + return nil + })) + + require.Equal(viaIter, viaScan, + "NewIter and Scan must observe the same key set and order") +} + +// ─── Flush / Close ────────────────────────────────────────────────────────── + +func (s *DBContractSuite) TestFlush_NoError() { + require := s.Require() + + require.NoError(s.db.Flush()) +} + +func (s *DBContractSuite) TestFlush_AfterWrites_DataStillReadable() { + require := s.Require() + + s.seed("persisted") + require.NoError(s.db.Flush()) + + val, closer, err := s.db.Get([]byte("persisted")) + require.NoError(err) + defer closer.Close() + require.Equal([]byte("val-persisted"), val) +} + +// ─── Large-ish workload ───────────────────────────────────────────────────── + +func (s *DBContractSuite) TestManyKeys_AllRetrievable() { + require := s.Require() + + const n = 500 + + b := s.db.NewBatch() + for i := 0; i < n; i++ { + k := fmt.Sprintf("key-%04d", i) + v := fmt.Sprintf("value-%04d", i) + require.NoError(b.Set([]byte(k), []byte(v))) + } + require.NoError(b.Commit(Sync)) + require.NoError(b.Close()) + + // Spot-check reads. + for _, i := range []int{0, n / 2, n - 1} { + k := fmt.Sprintf("key-%04d", i) + want := fmt.Sprintf("value-%04d", i) + val, closer, err := s.db.Get([]byte(k)) + require.NoError(err) + require.Equal([]byte(want), val) + closer.Close() + } + + // Full scan must visit exactly n keys. + count := 0 + require.NoError(s.db.Scan(nil, func(k, v []byte) error { + count++ + return nil + })) + require.Equal(n, count) +} diff --git a/internal/storage/engines_test.go b/internal/storage/engines_test.go new file mode 100644 index 0000000..b8cd948 --- /dev/null +++ b/internal/storage/engines_test.go @@ -0,0 +1,40 @@ +package storage + +import ( + "path/filepath" + "testing" + + "github.com/futureq-io/futureq/internal/config" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" +) + +// newPebbleEngine returns an in-memory Pebble instance for contract testing. +func newPebbleEngine(t *testing.T) DB { + t.Helper() + db, err := NewPebble(config.Pebble{DataPath: ""}, zap.NewNop()) + if err != nil { + t.Fatalf("pebble factory: %v", err) + } + return db +} + +// newBoltEngine returns a temp-file Bolt instance for contract testing. +func newBoltEngine(t *testing.T) DB { + t.Helper() + db, err := NewBoltDB(config.Bolt{ + DataPath: filepath.Join(t.TempDir(), "contract.db"), + }) + if err != nil { + t.Fatalf("bolt factory: %v", err) + } + return db +} + +func TestPebbleContract(t *testing.T) { + suite.Run(t, &DBContractSuite{NewEngine: newPebbleEngine}) +} + +func TestBoltContract(t *testing.T) { + suite.Run(t, &DBContractSuite{NewEngine: newBoltEngine}) +} diff --git a/internal/storage/interface.go b/internal/storage/interface.go deleted file mode 100644 index ca5facc..0000000 --- a/internal/storage/interface.go +++ /dev/null @@ -1,12 +0,0 @@ -package storage - -import ( - "time" -) - -type TaskStorage interface { - InitiatePersistence() error - Add(payload []byte, at time.Time) - PopLesserThan(v time.Time) []Task - LesserThan(v time.Time) []Task -} diff --git a/internal/storage/memory_array.go b/internal/storage/memory_array.go deleted file mode 100644 index 4a48219..0000000 --- a/internal/storage/memory_array.go +++ /dev/null @@ -1,158 +0,0 @@ -package storage - -import ( - "errors" - "fmt" - "sync" - "time" - - "github.com/cockroachdb/pebble" - "github.com/google/uuid" - - "github.com/futureq-io/futureq/internal/config" -) - -type memoryArray struct { - tasks Tasks - lock *sync.RWMutex - - cfg config.Persistence - db *pebble.DB -} - -func NewMemoryArray(cfg config.Persistence) TaskStorage { - return &memoryArray{ - //Tasks: make([]Task, 0), - lock: new(sync.RWMutex), - cfg: cfg, - } -} - -func (s *memoryArray) InitiatePersistence() error { - var err error - - s.db, err = pebble.Open(s.cfg.Path, &pebble.Options{}) - if err != nil { - return fmt.Errorf("could not open database: %v", err) - } - - return s.loadTasksFromDisk() -} - -func (s *memoryArray) Add(payload []byte, at time.Time) { - id := uuid.New().String() - t := Task{ID: id, At: at} - - s.lock.Lock() - defer s.lock.Unlock() - - s.tasks = append(s.tasks, t) - - for i := len(s.tasks) - 1; i > 0; i-- { - if s.tasks[i].At.Before(s.tasks[i-1].At) { - s.tasks[i], s.tasks[i-1] = s.tasks[i-1], s.tasks[i] - } - } - - err := s.saveTasksOnDisk() - if err != nil { - fmt.Println(err) - } - - err = s.db.Set([]byte(id), payload, pebble.Sync) - if err != nil { - fmt.Println(err) - } - -} - -func (s *memoryArray) PopLesserThan(v time.Time) []Task { - res, i := s.lesserThan(v) - s.popFromI(i) - - return res -} - -func (s *memoryArray) LesserThan(v time.Time) []Task { - res, _ := s.lesserThan(v) - - return res -} - -func (s *memoryArray) lesserThan(v time.Time) ([]Task, int) { - s.lock.RLock() - defer s.lock.RUnlock() - - result := make([]Task, 0) - - var i = 0 - - for ; i < len(s.tasks); i++ { - if s.tasks[i].At.After(v) { - break - } - - result = append(result, s.tasks[i]) - } - - for i = 0; i < len(result); i++ { - payload, closer, _ := s.db.Get([]byte(result[i].ID)) - _ = closer.Close() - - _ = s.db.Delete([]byte(result[i].ID), nil) - - result[i].Payload = payload - } - - return result, i -} - -func (s *memoryArray) popFromI(i int) { - s.lock.Lock() - defer s.lock.Unlock() - - s.tasks = s.tasks[i:] - - err := s.saveTasksOnDisk() - if err != nil { - fmt.Println(err) - } -} - -func (s *memoryArray) saveTasksOnDisk() error { - v, err := s.tasks.toGOB() - if err != nil { - return err - } - - err = s.db.Set([]byte("key"), v, pebble.Sync) - if err != nil { - return err - } - - return nil -} - -func (s *memoryArray) loadTasksFromDisk() error { - payload, closer, err := s.db.Get([]byte("key")) - if err != nil { - if !errors.Is(err, pebble.ErrNotFound) { - return err - } - - s.tasks = make([]Task, 0) - - return nil - } - - _ = closer.Close() - - fmt.Println("initiating from disk") - - s.tasks, err = FromGOB(payload) - if err != nil { - return err - } - - return nil -} diff --git a/internal/storage/pebble.go b/internal/storage/pebble.go new file mode 100644 index 0000000..cba8917 --- /dev/null +++ b/internal/storage/pebble.go @@ -0,0 +1,172 @@ +package storage + +import ( + "fmt" + "io" + + "github.com/cockroachdb/pebble/v2" + "github.com/cockroachdb/pebble/v2/vfs" + "github.com/futureq-io/futureq/internal/config" + "go.uber.org/zap" +) + +// Pebble holds an open pebble database and implements storage.DB. +type Pebble struct { + db *pebble.DB + logger *zap.Logger +} + +// NewPebble opens a Pebble database using the given config. +func NewPebble(cfg config.Pebble, logger *zap.Logger) (*Pebble, error) { + pebbleLogger := logger.Named("storage").With( + zap.String("engine", "pebble"), + ) + + cacheSize := cfg.CacheSizeMB * 1024 * 1024 + if cacheSize <= 0 { + cacheSize = 64 * 1024 * 1024 + } + + cache := pebble.NewCache(cacheSize) + // this somehow prevents memory leaks in the opts + defer cache.Unref() + + eventListener := pebble.MakeLoggingEventListener(pebbleLogger.Sugar()) + dbOpts := &pebble.Options{ + DisableWAL: cfg.DisableWAL, + Logger: pebbleLogger.Sugar(), + Cache: cache, + MemTableSize: cfg.InMemTableSizeMB * 1024 * 1024, + EventListener: &eventListener, + } + + if cfg.DataPath == "" { + dbOpts.FS = vfs.NewMem() + pebbleLogger.Info("Initializing Pebble DB in memory", zap.Bool("persist", false)) + } else { + pebbleLogger.Info("Initializing Pebble DB", zap.Bool("persist", true)) + } + + db, err := pebble.Open(cfg.DataPath, dbOpts) + if err != nil { + return nil, err + } + + return &Pebble{db: db, logger: pebbleLogger}, nil +} + +// ── storage.DB implementation ───────────────────────────────────────────────── + +func (p *Pebble) Get(key []byte) ([]byte, io.Closer, error) { + return p.db.Get(key) +} + +func (p *Pebble) NewBatch() Batch { + return &pebbleBatch{b: p.db.NewBatch()} +} + +// NewIter opens a snapshot internally and returns an iterator scoped to it. +// Close() on the returned iterator releases both the iterator and the snapshot. +func (p *Pebble) NewIter(opts *IterOptions) (Iterator, error) { + snap := p.db.NewSnapshot() + + var po *pebble.IterOptions + if opts != nil { + po = &pebble.IterOptions{ + LowerBound: opts.LowerBound, + UpperBound: opts.UpperBound, + } + } + + iter, err := snap.NewIter(po) + if err != nil { + _ = snap.Close() + return nil, err + } + + return &pebbleIterator{iter: iter, snap: snap}, nil +} + +func (p *Pebble) Scan(opts *IterOptions, yield func(key, value []byte) error) error { + snap := p.db.NewSnapshot() + + defer snap.Close() //nolint:errcheck + + var po *pebble.IterOptions + if opts != nil { + po = &pebble.IterOptions{ + LowerBound: opts.LowerBound, + UpperBound: opts.UpperBound, + } + } + + iter, err := snap.NewIter(po) + if err != nil { + return err + } + + defer iter.Close() //nolint:errcheck + + for iter.First(); iter.Valid(); iter.Next() { + if err := yield(iter.Key(), iter.Value()); err != nil { + return fmt.Errorf("yield func: %w", err) + } + } + + return iter.Error() +} + +func (p *Pebble) Flush() error { return p.db.Flush() } +func (p *Pebble) Close() error { return p.db.Close() } + +// ── pebbleBatch ─────────────────────────────────────────────────────────────── + +type pebbleBatch struct { + b *pebble.Batch +} + +func (pb *pebbleBatch) Set(key, value []byte) error { return pb.b.Set(key, value, nil) } +func (pb *pebbleBatch) Delete(key []byte) error { return pb.b.Delete(key, nil) } +func (pb *pebbleBatch) Close() error { return pb.b.Close() } + +func (pb *pebbleBatch) Commit(mode SyncMode) error { + switch mode { + case NoSync: + return pb.b.Commit(pebble.NoSync) + default: + return pb.b.Commit(pebble.Sync) + } +} + +// ── pebbleIterator ──────────────────────────────────────────────────────────── + +// pebbleIterator owns both the pebble.Iterator and the pebble.Snapshot it was +// created from. Close() releases both so callers only manage one resource. +type pebbleIterator struct { + iter *pebble.Iterator + snap *pebble.Snapshot +} + +func (pi *pebbleIterator) First() bool { return pi.iter.First() } +func (pi *pebbleIterator) Next() bool { return pi.iter.Next() } +func (pi *pebbleIterator) Valid() bool { return pi.iter.Valid() } +func (pi *pebbleIterator) Key() []byte { return pi.iter.Key() } +func (pi *pebbleIterator) Value() []byte { return pi.iter.Value() } +func (pi *pebbleIterator) Error() error { return pi.iter.Error() } +func (pi *pebbleIterator) Close() error { + iterErr := pi.iter.Close() + snapErr := pi.snap.Close() + if iterErr != nil { + return iterErr + } + + return snapErr +} + +// ── compile-time interface checks ───────────────────────────────────────────── + +var ( + _ DB = (*Pebble)(nil) + _ Batch = (*pebbleBatch)(nil) + _ Iterator = (*pebbleIterator)(nil) +) diff --git a/internal/storage/task.go b/internal/storage/task.go deleted file mode 100644 index ef33399..0000000 --- a/internal/storage/task.go +++ /dev/null @@ -1,43 +0,0 @@ -package storage - -import ( - "bytes" - "encoding/gob" - "fmt" - "time" -) - -type Task struct { - Payload []byte - ID string - At time.Time -} - -func (t *Task) String() string { - return fmt.Sprintf("%s-%s-%s", t.ID, string(t.Payload), t.At.String()) -} - -type Tasks []Task - -func (t Tasks) toGOB() ([]byte, error) { - var buffer bytes.Buffer - - err := gob.NewEncoder(&buffer). - Encode(t) - - return buffer.Bytes(), err -} - -func init() { - gob.Register(time.Time{}) - gob.Register(Tasks{}) -} - -func FromGOB(v []byte) (Tasks, error) { - var t Tasks - - err := gob.NewDecoder(bytes.NewReader(v)). - Decode(&t) - - return t, err -} diff --git a/internal/ticker/ticker.go b/internal/ticker/ticker.go deleted file mode 100644 index dcdfc59..0000000 --- a/internal/ticker/ticker.go +++ /dev/null @@ -1,34 +0,0 @@ -package ticker - -import ( - "time" - - "github.com/futureq-io/futureq/internal/q" - "github.com/futureq-io/futureq/internal/storage" -) - -type Ticker interface { - Tick() -} - -type ticker struct { - strg storage.TaskStorage - q q.Q -} - -func NewTicker(strg storage.TaskStorage, q q.Q) Ticker { - return &ticker{ - strg: strg, - q: q, - } -} - -func (t *ticker) Tick() { - ticker := time.NewTicker(1 * time.Second) - for tickedAt := range ticker.C { - result := t.strg.PopLesserThan(tickedAt) - for _, re := range result { - t.q.Publish(re.Payload) - } - } -} diff --git a/pkg/log/logger.go b/pkg/log/logger.go new file mode 100644 index 0000000..247d48f --- /dev/null +++ b/pkg/log/logger.go @@ -0,0 +1,31 @@ +package log + +import ( + "github.com/futureq-io/futureq/internal/config" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func InitLogger(cfg config.Logger) (*zap.Logger, error) { + zapConfig := zap.NewProductionConfig() + zapConfig.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + zapConfig.EncoderConfig.TimeKey = "timestamp" + + var level zapcore.Level + if err := level.UnmarshalText([]byte(cfg.Level)); err == nil { + zapConfig.Level = zap.NewAtomicLevelAt(level) + } + + logger, err := zapConfig.Build( + zap.AddCaller(), + zap.AddStacktrace(zapcore.ErrorLevel), + ) + + if err != nil { + return nil, err + } + + zap.ReplaceGlobals(logger) + + return logger, nil +} diff --git a/pkg/raft/metadata/commands.go b/pkg/raft/metadata/commands.go new file mode 100644 index 0000000..8ed52c7 --- /dev/null +++ b/pkg/raft/metadata/commands.go @@ -0,0 +1,232 @@ +package metadata + +import ( + "encoding/binary" + "fmt" +) + +// MetadataShardID is the reserved shard ID for the cluster metadata Raft group. +// This group is separate from any event data shard and is used exclusively for +// replicating cluster topology (leader info, membership, roles). +const MetadataShardID uint64 = 0 + +// CommandType identifies the metadata state machine operation. +type CommandType uint8 + +const ( + // UpdateTopologyCmd replaces the full topology snapshot for a shard. + // Sent when a leader changes or membership changes in any shard. + UpdateTopologyCmd CommandType = iota + + // RegisterNodeAddrCmd registers a node's client-facing gRPC address. + // Each node proposes this once at startup with its own address. + // The address registry is global (not per-shard) and survives + // topology updates — publishTopology merges it into each shard's + // GrpcAddrs map before proposing. + RegisterNodeAddrCmd +) + +// ShardTopology describes the current state of a single Raft shard. +// +// Nodes/NonVotings/Witnesses hold Raft addresses (from Dragonboat membership). +// GrpcAddrs holds each node's client-facing gRPC address — this is what SDKs +// use to connect to the leader. +type ShardTopology struct { + ShardID uint64 + LeaderID uint64 + // LeaderAddr is the leader's gRPC address (client-facing). + LeaderAddr string + Term uint64 + Epoch uint64 // incremented on every topology change + Nodes map[uint64]string + NonVotings map[uint64]string + Witnesses map[uint64]string + GrpcAddrs map[uint64]string // nodeID → gRPC address + ConfigChangeID uint64 +} + +// TopologySnapshot is the full cluster topology across all shards. +type TopologySnapshot struct { + Shards map[uint64]*ShardTopology + Epoch uint64 // global epoch, incremented on any change +} + +// MarshalUpdateTopologyCmd serialises a ShardTopology into a command payload. +// +// Wire format: +// +// [0] CommandType (1 byte = 0) +// [1..8] ShardID (uint64 big-endian) +// [9..16] LeaderID (uint64 big-endian) +// [17..24] Term (uint64 big-endian) +// [25..32] Epoch (uint64 big-endian) +// [33..40] ConfigChangeID (uint64 big-endian) +// [41..42] LeaderAddrLen (uint16 big-endian) +// [43..] LeaderAddr (variable — gRPC address) +// [..+2] NumNodes (uint16 big-endian) +// for each node: +// [n..n+7] NodeID (uint64 big-endian) +// [n+8..n+9] AddrLen (uint16 big-endian) +// [n+10..] Addr (variable — Raft address) +// [..+2] NumNonVotings (uint16 big-endian) +// for each non-voting: same as node +// [..+2] NumWitnesses (uint16 big-endian) +// for each witness: same as node +// [..+2] NumGrpcAddrs (uint16 big-endian) +// for each grpc addr: same as node (gRPC address) +func MarshalUpdateTopologyCmd(t *ShardTopology) ([]byte, error) { + size := 1 + 8*5 + 2 + len(t.LeaderAddr) + for _, m := range []map[uint64]string{t.Nodes, t.NonVotings, t.Witnesses, t.GrpcAddrs} { + size += 2 + for _, addr := range m { + size += 8 + 2 + len(addr) + } + } + + out := make([]byte, size) + out[0] = byte(UpdateTopologyCmd) + pos := 1 + + binary.BigEndian.PutUint64(out[pos:], t.ShardID) + pos += 8 + binary.BigEndian.PutUint64(out[pos:], t.LeaderID) + pos += 8 + binary.BigEndian.PutUint64(out[pos:], t.Term) + pos += 8 + binary.BigEndian.PutUint64(out[pos:], t.Epoch) + pos += 8 + binary.BigEndian.PutUint64(out[pos:], t.ConfigChangeID) + pos += 8 + + binary.BigEndian.PutUint16(out[pos:], uint16(len(t.LeaderAddr))) + pos += 2 + copy(out[pos:], t.LeaderAddr) + pos += len(t.LeaderAddr) + + pos = marshalNodeMap(out, pos, t.Nodes) + pos = marshalNodeMap(out, pos, t.NonVotings) + pos = marshalNodeMap(out, pos, t.Witnesses) + _ = marshalNodeMap(out, pos, t.GrpcAddrs) + + return out, nil +} + +// UnmarshalUpdateTopologyCmd deserialises an UpdateTopologyCmd payload. +func UnmarshalUpdateTopologyCmd(data []byte) (*ShardTopology, error) { + if len(data) < 1+8*5+2 { + return nil, fmt.Errorf("metadata: UpdateTopologyCmd too short: %d bytes", len(data)) + } + if CommandType(data[0]) != UpdateTopologyCmd { + return nil, fmt.Errorf("metadata: expected UpdateTopologyCmd (0), got %d", data[0]) + } + + t := &ShardTopology{} + pos := 1 + + t.ShardID = binary.BigEndian.Uint64(data[pos:]) + pos += 8 + t.LeaderID = binary.BigEndian.Uint64(data[pos:]) + pos += 8 + t.Term = binary.BigEndian.Uint64(data[pos:]) + pos += 8 + t.Epoch = binary.BigEndian.Uint64(data[pos:]) + pos += 8 + t.ConfigChangeID = binary.BigEndian.Uint64(data[pos:]) + pos += 8 + + addrLen := int(binary.BigEndian.Uint16(data[pos:])) + pos += 2 + if pos+addrLen > len(data) { + return nil, fmt.Errorf("metadata: UpdateTopologyCmd truncated at LeaderAddr") + } + t.LeaderAddr = string(data[pos : pos+addrLen]) + pos += addrLen + + var err error + if t.Nodes, pos, err = unmarshalNodeMap(data, pos); err != nil { + return nil, fmt.Errorf("metadata: nodes: %w", err) + } + if t.NonVotings, pos, err = unmarshalNodeMap(data, pos); err != nil { + return nil, fmt.Errorf("metadata: nonVotings: %w", err) + } + if t.Witnesses, pos, err = unmarshalNodeMap(data, pos); err != nil { + return nil, fmt.Errorf("metadata: witnesses: %w", err) + } + if t.GrpcAddrs, _, err = unmarshalNodeMap(data, pos); err != nil { + return nil, fmt.Errorf("metadata: grpcAddrs: %w", err) + } + + return t, nil +} + +func marshalNodeMap(out []byte, pos int, m map[uint64]string) int { + binary.BigEndian.PutUint16(out[pos:], uint16(len(m))) + pos += 2 + for id, addr := range m { + binary.BigEndian.PutUint64(out[pos:], id) + pos += 8 + binary.BigEndian.PutUint16(out[pos:], uint16(len(addr))) + pos += 2 + copy(out[pos:], addr) + pos += len(addr) + } + return pos +} + +func unmarshalNodeMap(data []byte, pos int) (map[uint64]string, int, error) { + if pos+2 > len(data) { + return nil, 0, fmt.Errorf("truncated at count") + } + count := int(binary.BigEndian.Uint16(data[pos:])) + pos += 2 + + m := make(map[uint64]string, count) + for i := 0; i < count; i++ { + if pos+8+2 > len(data) { + return nil, 0, fmt.Errorf("truncated at entry %d", i) + } + id := binary.BigEndian.Uint64(data[pos:]) + pos += 8 + addrLen := int(binary.BigEndian.Uint16(data[pos:])) + pos += 2 + if pos+addrLen > len(data) { + return nil, 0, fmt.Errorf("truncated at entry %d addr", i) + } + m[id] = string(data[pos : pos+addrLen]) + pos += addrLen + } + return m, pos, nil +} + +// MarshalRegisterNodeAddrCmd serialises a (nodeID, grpcAddr) registration. +// +// Wire format: +// +// [0] CommandType (1 byte = 1) +// [1..8] NodeID (uint64 big-endian) +// [9..10] AddrLen (uint16 big-endian) +// [11..] Addr (variable — gRPC address) +func MarshalRegisterNodeAddrCmd(nodeID uint64, grpcAddr string) ([]byte, error) { + out := make([]byte, 1+8+2+len(grpcAddr)) + out[0] = byte(RegisterNodeAddrCmd) + binary.BigEndian.PutUint64(out[1:], nodeID) + binary.BigEndian.PutUint16(out[9:], uint16(len(grpcAddr))) + copy(out[11:], grpcAddr) + return out, nil +} + +// UnmarshalRegisterNodeAddrCmd deserialises a RegisterNodeAddrCmd payload. +func UnmarshalRegisterNodeAddrCmd(data []byte) (nodeID uint64, grpcAddr string, err error) { + if len(data) < 1+8+2 { + return 0, "", fmt.Errorf("metadata: RegisterNodeAddrCmd too short: %d bytes", len(data)) + } + if CommandType(data[0]) != RegisterNodeAddrCmd { + return 0, "", fmt.Errorf("metadata: expected RegisterNodeAddrCmd (1), got %d", data[0]) + } + nodeID = binary.BigEndian.Uint64(data[1:]) + addrLen := int(binary.BigEndian.Uint16(data[9:])) + if 11+addrLen > len(data) { + return 0, "", fmt.Errorf("metadata: RegisterNodeAddrCmd truncated at addr") + } + return nodeID, string(data[11 : 11+addrLen]), nil +} diff --git a/pkg/raft/metadata/commands_test.go b/pkg/raft/metadata/commands_test.go new file mode 100644 index 0000000..9bfeed5 --- /dev/null +++ b/pkg/raft/metadata/commands_test.go @@ -0,0 +1,172 @@ +package metadata + +import ( + "bytes" + "fmt" + "testing" + + "github.com/stretchr/testify/suite" +) + +type CommandsSuite struct { + suite.Suite +} + +func TestCommandsSuite(t *testing.T) { + suite.Run(t, new(CommandsSuite)) +} + +// ─── UpdateTopologyCmd round-trips ────────────────────────────────────────── + +func (s *CommandsSuite) TestUpdateTopologyCmd_BasicRoundTrip() { + require := s.Require() + + orig := &ShardTopology{ + ShardID: 7, + LeaderID: 2, + LeaderAddr: "10.0.0.1:9000", + Term: 42, + Epoch: 100, + ConfigChangeID: 555, + Nodes: map[uint64]string{1: "raft://1", 2: "raft://2"}, + NonVotings: map[uint64]string{3: "raft://3"}, + Witnesses: map[uint64]string{}, + GrpcAddrs: map[uint64]string{1: "grpc://1", 2: "grpc://2"}, + } + + cmd, err := MarshalUpdateTopologyCmd(orig) + require.NoError(err) + require.Equal(byte(UpdateTopologyCmd), cmd[0]) + + got, err := UnmarshalUpdateTopologyCmd(cmd) + require.NoError(err) + + require.Equal(orig.ShardID, got.ShardID) + require.Equal(orig.LeaderID, got.LeaderID) + require.Equal(orig.LeaderAddr, got.LeaderAddr) + require.Equal(orig.Term, got.Term) + require.Equal(orig.Epoch, got.Epoch) + require.Equal(orig.ConfigChangeID, got.ConfigChangeID) + require.Equal(orig.Nodes, got.Nodes) + require.Equal(orig.NonVotings, got.NonVotings) + require.Equal(orig.Witnesses, got.Witnesses) + require.Equal(orig.GrpcAddrs, got.GrpcAddrs) +} + +func (s *CommandsSuite) TestUpdateTopologyCmd_EmptyMaps() { + require := s.Require() + + orig := &ShardTopology{ + ShardID: 1, + Nodes: map[uint64]string{}, + NonVotings: map[uint64]string{}, + Witnesses: map[uint64]string{}, + GrpcAddrs: map[uint64]string{}, + } + + cmd, err := MarshalUpdateTopologyCmd(orig) + require.NoError(err) + + got, err := UnmarshalUpdateTopologyCmd(cmd) + require.NoError(err) + require.Equal(orig.ShardID, got.ShardID) + require.Empty(got.Nodes) + require.Empty(got.NonVotings) +} + +func (s *CommandsSuite) TestUpdateTopologyCmd_WrongType_Fails() { + require := s.Require() + + // Build a RegisterNodeAddrCmd but try to parse as UpdateTopologyCmd. + regCmd, _ := MarshalRegisterNodeAddrCmd(1, "addr") + regCmd[0] = byte(RegisterNodeAddrCmd) + + _, err := UnmarshalUpdateTopologyCmd(regCmd) + require.Error(err) +} + +func (s *CommandsSuite) TestUpdateTopologyCmd_TooShort_Fails() { + require := s.Require() + + _, err := UnmarshalUpdateTopologyCmd([]byte{0x00, 0x01}) + require.Error(err) +} + +func (s *CommandsSuite) TestUpdateTopologyCmd_TruncatedLeaderAddr_Fails() { + require := s.Require() + + orig := &ShardTopology{ + ShardID: 1, + LeaderAddr: "a-very-long-hostname.example.com:9999", + Nodes: map[uint64]string{}, + } + cmd, _ := MarshalUpdateTopologyCmd(orig) + + // Cut the buffer before the leaderAddr data ends. + _, err := UnmarshalUpdateTopologyCmd(cmd[:40]) + require.Error(err) +} + +// ─── RegisterNodeAddrCmd round-trips ─────────────────────────────────────── + +func (s *CommandsSuite) TestRegisterNodeAddrCmd_BasicRoundTrip() { + require := s.Require() + + nodeID := uint64(9) + addr := "192.168.1.10:8443" + + cmd, err := MarshalRegisterNodeAddrCmd(nodeID, addr) + require.NoError(err) + require.Equal(byte(RegisterNodeAddrCmd), cmd[0]) + + gotID, gotAddr, err := UnmarshalRegisterNodeAddrCmd(cmd) + require.NoError(err) + require.Equal(nodeID, gotID) + require.Equal(addr, gotAddr) +} + +func (s *CommandsSuite) TestRegisterNodeAddrCmd_EmptyAddr() { + require := s.Require() + + cmd, err := MarshalRegisterNodeAddrCmd(1, "") + require.NoError(err) + + _, gotAddr, err := UnmarshalRegisterNodeAddrCmd(cmd) + require.NoError(err) + require.Equal("", gotAddr) +} + +func (s *CommandsSuite) TestRegisterNodeAddrCmd_WrongType_Fails() { + require := s.Require() + + // Topology command but try as RegisterNodeAddrCmd. + topo := &ShardTopology{ShardID: 1, Nodes: map[uint64]string{}} + topoCmd, _ := MarshalUpdateTopologyCmd(topo) + + _, _, err := UnmarshalRegisterNodeAddrCmd(topoCmd) + require.Error(err) +} + +func (s *CommandsSuite) TestRegisterNodeAddrCmd_TooShort_Fails() { + require := s.Require() + + _, _, err := UnmarshalRegisterNodeAddrCmd([]byte{0x01, 0x00, 0x00}) + require.Error(err) +} + +// ─── Metadata snapshot round-trips (via exported helpers) ────────────────── + +func (s *CommandsSuite) TestWriteReadUint32_RoundTrip() { + require := s.Require() + + values := []uint32{0, 1, 127, 256, 65535, 1 << 20, 0xFFFFFFFF} + var buf bytes.Buffer + for _, v := range values { + require.NoError(writeUint32(&buf, v)) + } + for _, expected := range values { + got, err := readUint32(&buf) + require.NoError(err) + require.Equal(expected, got, fmt.Sprintf("expected %d, got %d", expected, got)) + } +} diff --git a/pkg/raft/metadata/service.go b/pkg/raft/metadata/service.go new file mode 100644 index 0000000..b3d5d86 --- /dev/null +++ b/pkg/raft/metadata/service.go @@ -0,0 +1,284 @@ +package metadata + +import ( + "context" + "sync" + "time" + + "github.com/lni/dragonboat/v4" + "github.com/lni/dragonboat/v4/raftio" + "go.uber.org/zap" +) + +// Service watches Dragonboat for leader and membership changes across all +// shards and replicates topology updates through the metadata Raft group. +// +// It implements raftio.IRaftEventListener (leader changes) and +// raftio.ISystemEventListener (membership changes). Register it on +// NodeHostConfig so Dragonboat calls it on every event. +type Service struct { + nh *dragonboat.NodeHost + logger *zap.Logger + + // propose submits a command to the metadata Raft group. + propose func(ctx context.Context, cmd []byte) error + + // getGrpcAddrs returns the current nodeID → gRPC address registry + // from the metadata state machine. Set via SetGrpcAddrsSource. + getGrpcAddrs func() map[uint64]string + + mu sync.Mutex + epoch uint64 + shards map[uint64]struct{} // tracks which shards we know about +} + +// NewService creates a metadata Service. The propose function should submit +// a command to the metadata Raft group via NodeHost.SyncPropose. +// nh may be nil during construction — call SetNodeHost before the service +// handles any events. +func NewService(nh *dragonboat.NodeHost, propose func(ctx context.Context, cmd []byte) error, logger *zap.Logger) *Service { + return &Service{ + nh: nh, + propose: propose, + logger: logger.Named("metadata_svc"), + shards: make(map[uint64]struct{}), + } +} + +// SetNodeHost sets the NodeHost reference. Must be called before the service +// handles any Dragonboat events. +func (s *Service) SetNodeHost(nh *dragonboat.NodeHost) { + s.mu.Lock() + defer s.mu.Unlock() + s.nh = nh +} + +// SetGrpcAddrsSource sets the function used to read the current gRPC +// address registry. Called during publishTopology to merge registered +// addresses into each shard's GrpcAddrs map. +func (s *Service) SetGrpcAddrsSource(fn func() map[uint64]string) { + s.mu.Lock() + defer s.mu.Unlock() + s.getGrpcAddrs = fn +} + +// RegisterNodeAddr proposes this node's gRPC address to the metadata group. +// Call once at startup after the metadata group is running. +func (s *Service) RegisterNodeAddr(ctx context.Context, nodeID uint64, grpcAddr string) error { + cmd, err := MarshalRegisterNodeAddrCmd(nodeID, grpcAddr) + if err != nil { + return err + } + if err := s.propose(ctx, cmd); err != nil { + return err + } + s.logger.Info("registered node gRPC address", + zap.Uint64("node_id", nodeID), + zap.String("grpc_addr", grpcAddr), + ) + return nil +} + +// ─── IRaftEventListener ────────────────────────────────────────────────────── + +// LeaderUpdated is called by Dragonboat when a leader changes for any shard. +func (s *Service) LeaderUpdated(info raftio.LeaderInfo) { + if info.ShardID == MetadataShardID { + return // don't track the metadata shard itself + } + + s.logger.Info("leader updated", + zap.Uint64("shard_id", info.ShardID), + zap.Uint64("leader_id", info.LeaderID), + zap.Uint64("term", info.Term), + ) + + s.mu.Lock() + s.shards[info.ShardID] = struct{}{} + s.mu.Unlock() + + s.publishTopology(info.ShardID) +} + +// ─── ISystemEventListener ──────────────────────────────────────────────────── + +// MembershipChanged is called by Dragonboat when membership changes for any shard. +func (s *Service) MembershipChanged(info raftio.NodeInfo) { + if info.ShardID == MetadataShardID { + return + } + + s.logger.Info("membership changed", + zap.Uint64("shard_id", info.ShardID), + zap.Uint64("replica_id", info.ReplicaID), + ) + + s.mu.Lock() + s.shards[info.ShardID] = struct{}{} + s.mu.Unlock() + + s.publishTopology(info.ShardID) +} + +// NodeHostShuttingDown is called when the NodeHost is shutting down. +func (s *Service) NodeHostShuttingDown() {} + +// NodeUnloaded is called when a shard replica is unloaded. +func (s *Service) NodeUnloaded(info raftio.NodeInfo) {} + +// NodeDeleted is called when a shard replica is deleted. +func (s *Service) NodeDeleted(info raftio.NodeInfo) { + if info.ShardID == MetadataShardID { + return + } + s.publishTopology(info.ShardID) +} + +// NodeReady is called when a shard replica is ready. +func (s *Service) NodeReady(info raftio.NodeInfo) { + if info.ShardID == MetadataShardID { + return + } + s.mu.Lock() + s.shards[info.ShardID] = struct{}{} + s.mu.Unlock() + s.publishTopology(info.ShardID) +} + +// ConnectionEstablished is called when a connection is established. +func (s *Service) ConnectionEstablished(info raftio.ConnectionInfo) {} + +// ConnectionFailed is called when a connection attempt fails. +func (s *Service) ConnectionFailed(info raftio.ConnectionInfo) {} + +// SendSnapshotStarted is called when sending a snapshot starts. +func (s *Service) SendSnapshotStarted(info raftio.SnapshotInfo) {} + +// SendSnapshotCompleted is called when sending a snapshot completes. +func (s *Service) SendSnapshotCompleted(info raftio.SnapshotInfo) {} + +// SendSnapshotAborted is called when sending a snapshot is aborted. +func (s *Service) SendSnapshotAborted(info raftio.SnapshotInfo) {} + +// SnapshotReceived is called when a snapshot is received. +func (s *Service) SnapshotReceived(info raftio.SnapshotInfo) {} + +// SnapshotRecovered is called when snapshot recovery completes. +func (s *Service) SnapshotRecovered(info raftio.SnapshotInfo) {} + +// SnapshotCreated is called when a snapshot is created. +func (s *Service) SnapshotCreated(info raftio.SnapshotInfo) {} + +// SnapshotCompacted is called when a snapshot is compacted. +func (s *Service) SnapshotCompacted(info raftio.SnapshotInfo) {} + +// LogCompacted is called when the Raft log is compacted. +func (s *Service) LogCompacted(info raftio.EntryInfo) {} + +// LogDBCompacted is called when the LogDB is compacted. +func (s *Service) LogDBCompacted(info raftio.EntryInfo) {} + +// ─── Topology Publishing ──────────────────────────────────────────────────── + +// publishTopology queries Dragonboat for the current shard topology and +// proposes it to the metadata Raft group. +func (s *Service) publishTopology(shardID uint64) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Get membership info. + membership, err := s.nh.SyncGetShardMembership(ctx, shardID) + if err != nil { + s.logger.Error("failed to get shard membership", + zap.Uint64("shard_id", shardID), + zap.Error(err), + ) + return + } + + // Get leader info. + leaderID, term, valid, err := s.nh.GetLeaderID(shardID) + if err != nil || !valid { + leaderID = 0 + term = 0 + } + + // Merge the gRPC address registry (populated by RegisterNodeAddrCmd). + s.mu.Lock() + getAddrs := s.getGrpcAddrs + s.mu.Unlock() + grpcAddrs := make(map[uint64]string) + if getAddrs != nil { + for k, v := range getAddrs() { + grpcAddrs[k] = v + } + } + + // Resolve leader gRPC address (what clients dial). + leaderAddr := "" + if leaderID > 0 { + leaderAddr = grpcAddrs[leaderID] + } + + s.mu.Lock() + s.epoch++ + epoch := s.epoch + s.mu.Unlock() + + topo := &ShardTopology{ + ShardID: shardID, + LeaderID: leaderID, + LeaderAddr: leaderAddr, + Term: term, + Epoch: epoch, + ConfigChangeID: membership.ConfigChangeID, + Nodes: membership.Nodes, + NonVotings: membership.NonVotings, + Witnesses: membership.Witnesses, + GrpcAddrs: grpcAddrs, + } + + cmd, err := MarshalUpdateTopologyCmd(topo) + if err != nil { + s.logger.Error("failed to marshal topology command", zap.Error(err)) + return + } + + if err := s.propose(ctx, cmd); err != nil { + s.logger.Error("failed to propose topology update", + zap.Uint64("shard_id", shardID), + zap.Error(err), + ) + return + } + + s.logger.Debug("published topology", + zap.Uint64("shard_id", shardID), + zap.Uint64("leader_id", leaderID), + zap.Uint64("epoch", epoch), + ) +} + +// RefreshAll re-publishes topology for all known shards. +// Useful after startup to ensure the metadata group has the latest state. +func (s *Service) RefreshAll() { + s.mu.Lock() + shardIDs := make([]uint64, 0, len(s.shards)) + for id := range s.shards { + shardIDs = append(shardIDs, id) + } + s.mu.Unlock() + + for _, id := range shardIDs { + s.publishTopology(id) + } +} + +// RegisterShard explicitly adds a shard to the tracking set. +// Called during startup for shards that may not have fired events yet. +func (s *Service) RegisterShard(shardID uint64) { + s.mu.Lock() + s.shards[shardID] = struct{}{} + s.mu.Unlock() + s.publishTopology(shardID) +} diff --git a/pkg/raft/metadata/service_test.go b/pkg/raft/metadata/service_test.go new file mode 100644 index 0000000..2e96626 --- /dev/null +++ b/pkg/raft/metadata/service_test.go @@ -0,0 +1,174 @@ +package metadata + +import ( + "context" + "errors" + "testing" + + "github.com/lni/dragonboat/v4/raftio" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" +) + +type ServiceSuite struct { + suite.Suite +} + +func TestServiceSuite(t *testing.T) { + suite.Run(t, new(ServiceSuite)) +} + +// proposeOK is a fake propose function that captures the marshalled command. +type capturedPropose struct { + cmds [][]byte + err error +} + +func (c *capturedPropose) propose(_ context.Context, cmd []byte) error { + if c.err != nil { + return c.err + } + cp := make([]byte, len(cmd)) + copy(cp, cmd) + c.cmds = append(c.cmds, cp) + return nil +} + +// ─── Constructor / setters ────────────────────────────────────────────────── + +func (s *ServiceSuite) TestNewService_InitializesShardsMap() { + require := s.Require() + + cp := &capturedPropose{} + svc := NewService(nil, cp.propose, zap.NewNop()) + + require.NotNil(svc) + require.NotNil(svc.shards) + require.Empty(svc.shards) +} + +func (s *ServiceSuite) TestSetNodeHost_StoresReference() { + require := s.Require() + + cp := &capturedPropose{} + svc := NewService(nil, cp.propose, zap.NewNop()) + + // Verify SetNodeHost doesn't panic with a non-nil host. + svc.SetNodeHost(nil) + require.Nil(svc.nh) +} + +func (s *ServiceSuite) TestSetGrpcAddrsSource_StoresFunc() { + require := s.Require() + + cp := &capturedPropose{} + svc := NewService(nil, cp.propose, zap.NewNop()) + + fn := func() map[uint64]string { return map[uint64]string{1: "addr"} } + svc.SetGrpcAddrsSource(fn) + + svc.mu.Lock() + got := svc.getGrpcAddrs + svc.mu.Unlock() + + require.NotNil(got) + require.Equal(map[uint64]string{1: "addr"}, got()) +} + +// ─── RegisterNodeAddr ─────────────────────────────────────────────────────── + +func (s *ServiceSuite) TestRegisterNodeAddr_ProposesRegisterNodeAddrCmd() { + require := s.Require() + + cp := &capturedPropose{} + svc := NewService(nil, cp.propose, zap.NewNop()) + + err := svc.RegisterNodeAddr(context.Background(), 7, "node7:9000") + require.NoError(err) + require.Len(cp.cmds, 1) + + nodeID, addr, err := UnmarshalRegisterNodeAddrCmd(cp.cmds[0]) + require.NoError(err) + require.Equal(uint64(7), nodeID) + require.Equal("node7:9000", addr) +} + +func (s *ServiceSuite) TestRegisterNodeAddr_ProposeFails_ErrorPropagates() { + require := s.Require() + + sentinel := errors.New("propose failed") + cp := &capturedPropose{err: sentinel} + svc := NewService(nil, cp.propose, zap.NewNop()) + + err := svc.RegisterNodeAddr(context.Background(), 7, "node7:9000") + require.Error(err) + require.Contains(err.Error(), sentinel.Error()) +} + +// ─── LeaderUpdated / MembershipChanged ────────────────────────────────────── + +func (s *ServiceSuite) TestLeaderUpdated_IgnoresMetadataShard() { + require := s.Require() + + cp := &capturedPropose{} + svc := NewService(nil, cp.propose, zap.NewNop()) + + // MetadataShardID = 0 — should not trigger publishTopology. + svc.LeaderUpdated(raftio.LeaderInfo{ShardID: MetadataShardID, LeaderID: 1}) + + svc.mu.Lock() + defer svc.mu.Unlock() + require.Empty(svc.shards, "metadata shard must not be tracked") +} + +// ─── No-op event handlers ─────────────────────────────────────────────────── + +func (s *ServiceSuite) TestNoOpHandlers_DoNotPanic() { + require := s.Require() + + cp := &capturedPropose{} + svc := NewService(nil, cp.propose, zap.NewNop()) + + nodeInfo := raftio.NodeInfo{ShardID: 1} + connInfo := raftio.ConnectionInfo{} + snapInfo := raftio.SnapshotInfo{} + entryInfo := raftio.EntryInfo{} + + // None of these should panic or change state. + svc.NodeHostShuttingDown() + svc.NodeUnloaded(nodeInfo) + svc.ConnectionEstablished(connInfo) + svc.ConnectionFailed(connInfo) + svc.SendSnapshotStarted(snapInfo) + svc.SendSnapshotCompleted(snapInfo) + svc.SendSnapshotAborted(snapInfo) + svc.SnapshotReceived(snapInfo) + svc.SnapshotRecovered(snapInfo) + svc.SnapshotCreated(snapInfo) + svc.SnapshotCompacted(snapInfo) + svc.LogCompacted(entryInfo) + svc.LogDBCompacted(entryInfo) + + svc.mu.Lock() + defer svc.mu.Unlock() + require.Empty(svc.shards, "no-op handlers must not track shards") +} + +// ─── RegisterShard / RefreshAll ───────────────────────────────────────────── + +// RegisterShard and RefreshAll call publishTopology, which requires a non-nil +// NodeHost. We skip direct calls here — those paths are covered by integration +// tests against a real dragonboat cluster. These handlers verify the tracking +// semantics that CAN be tested without a NodeHost. + +func (s *ServiceSuite) TestRefreshAll_NoShards_DoesNothing() { + require := s.Require() + + cp := &capturedPropose{} + svc := NewService(nil, cp.propose, zap.NewNop()) + + // With no registered shards, RefreshAll must not call publishTopology + // (which would need a NodeHost). This must not panic. + svc.RefreshAll() + require.Empty(cp.cmds) +} diff --git a/pkg/raft/metadata/statemachine.go b/pkg/raft/metadata/statemachine.go new file mode 100644 index 0000000..7c8327a --- /dev/null +++ b/pkg/raft/metadata/statemachine.go @@ -0,0 +1,305 @@ +package metadata + +import ( + "io" + "sync" + + "github.com/lni/dragonboat/v4/statemachine" + "go.uber.org/zap" +) + +// MetadataStateMachine implements statemachine.IStateMachine (in-memory). +// It stores the cluster topology — per-shard leader info, membership, and roles. +// It also keeps a global nodeID → gRPC address registry populated by +// RegisterNodeAddrCmd; publishTopology merges this into each shard's GrpcAddrs. +// State is fully transient: rebuilt from the Raft log on restart. +type MetadataStateMachine struct { + mu sync.RWMutex + topology *TopologySnapshot + grpcAddrs map[uint64]string // nodeID → client-facing gRPC address + logger *zap.Logger +} + +// NewMetadataStateMachineFactory returns the factory function that Dragonboat +// passes (clusterID, nodeID) to when it instantiates a new replica. +func NewMetadataStateMachineFactory(logger *zap.Logger) func(uint64, uint64) statemachine.IStateMachine { + return func(clusterID, nodeID uint64) statemachine.IStateMachine { + return &MetadataStateMachine{ + topology: &TopologySnapshot{ + Shards: make(map[uint64]*ShardTopology), + }, + grpcAddrs: make(map[uint64]string), + logger: logger.Named("metadata_sm"), + } + } +} + +// Update applies a single Raft log entry to the in-memory state. +func (s *MetadataStateMachine) Update(entry statemachine.Entry) (statemachine.Result, error) { + if len(entry.Cmd) == 0 { + return statemachine.Result{Value: 0}, nil + } + + switch CommandType(entry.Cmd[0]) { + case UpdateTopologyCmd: + topo, err := UnmarshalUpdateTopologyCmd(entry.Cmd) + if err != nil { + s.logger.Error("failed to unmarshal UpdateTopologyCmd", zap.Error(err)) + return statemachine.Result{Value: 0}, nil + } + s.mu.Lock() + s.topology.Shards[topo.ShardID] = topo + if topo.Epoch > s.topology.Epoch { + s.topology.Epoch = topo.Epoch + } + s.mu.Unlock() + + s.logger.Debug("topology updated", + zap.Uint64("shard_id", topo.ShardID), + zap.Uint64("leader_id", topo.LeaderID), + zap.Uint64("epoch", topo.Epoch), + ) + return statemachine.Result{Value: 1}, nil + + case RegisterNodeAddrCmd: + nodeID, grpcAddr, err := UnmarshalRegisterNodeAddrCmd(entry.Cmd) + if err != nil { + s.logger.Error("failed to unmarshal RegisterNodeAddrCmd", zap.Error(err)) + return statemachine.Result{Value: 0}, nil + } + s.mu.Lock() + s.grpcAddrs[nodeID] = grpcAddr + s.mu.Unlock() + + s.logger.Debug("registered node gRPC address", + zap.Uint64("node_id", nodeID), + zap.String("grpc_addr", grpcAddr), + ) + return statemachine.Result{Value: 1}, nil + + default: + s.logger.Warn("unknown metadata command type", zap.Uint8("type", entry.Cmd[0])) + return statemachine.Result{Value: 0}, nil + } +} + +// Lookup handles read-only queries against the in-memory state. +// Supported query types: +// - nil or "topology": returns a copy of the full TopologySnapshot +// - uint64: returns the ShardTopology for that shard ID +func (s *MetadataStateMachine) Lookup(query interface{}) (interface{}, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + switch q := query.(type) { + case nil: + return s.copyTopology(), nil + case string: + if q == "topology" { + return s.copyTopology(), nil + } + case uint64: + if shard, ok := s.topology.Shards[q]; ok { + return copyShardTopology(shard), nil + } + return nil, nil + } + + return nil, nil +} + +// GetTopology returns a copy of the current topology snapshot. +// Safe for concurrent use. +func (s *MetadataStateMachine) GetTopology() *TopologySnapshot { + s.mu.RLock() + defer s.mu.RUnlock() + return s.copyTopology() +} + +// GetShardTopology returns a copy of the topology for a specific shard. +// Returns nil if the shard is not tracked. +func (s *MetadataStateMachine) GetShardTopology(shardID uint64) *ShardTopology { + s.mu.RLock() + defer s.mu.RUnlock() + if shard, ok := s.topology.Shards[shardID]; ok { + return copyShardTopology(shard) + } + return nil +} + +// GetGrpcAddrs returns a copy of the global nodeID → gRPC address registry. +// Safe for concurrent use. +func (s *MetadataStateMachine) GetGrpcAddrs() map[uint64]string { + s.mu.RLock() + defer s.mu.RUnlock() + cp := make(map[uint64]string, len(s.grpcAddrs)) + for k, v := range s.grpcAddrs { + cp[k] = v + } + return cp +} + +// SaveSnapshot serialises the in-memory state to the writer. +// For an in-memory state machine, this is used by Dragonboat to +// transfer state to new members joining the metadata group. +func (s *MetadataStateMachine) SaveSnapshot(w io.Writer, _ statemachine.ISnapshotFileCollection, _ <-chan struct{}) error { + s.mu.RLock() + defer s.mu.RUnlock() + + // Write the gRPC address registry first. + if err := writeUint32(w, uint32(len(s.grpcAddrs))); err != nil { + return err + } + for nodeID, addr := range s.grpcAddrs { + cmd, err := MarshalRegisterNodeAddrCmd(nodeID, addr) + if err != nil { + return err + } + if err := writeUint32(w, uint32(len(cmd))); err != nil { + return err + } + if _, err := w.Write(cmd); err != nil { + return err + } + } + + // Write number of shards. + count := uint32(len(s.topology.Shards)) + if err := writeUint32(w, count); err != nil { + return err + } + + for _, shard := range s.topology.Shards { + cmd, err := MarshalUpdateTopologyCmd(shard) + if err != nil { + return err + } + // Write command length + command bytes. + if err := writeUint32(w, uint32(len(cmd))); err != nil { + return err + } + if _, err := w.Write(cmd); err != nil { + return err + } + } + + return nil +} + +// RecoverFromSnapshot rebuilds the in-memory state from a snapshot reader. +func (s *MetadataStateMachine) RecoverFromSnapshot(r io.Reader, _ []statemachine.SnapshotFile, _ <-chan struct{}) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.topology = &TopologySnapshot{ + Shards: make(map[uint64]*ShardTopology), + } + s.grpcAddrs = make(map[uint64]string) + + // Read the gRPC address registry. + grpcCount, err := readUint32(r) + if err != nil { + return err + } + for i := uint32(0); i < grpcCount; i++ { + cmdLen, err := readUint32(r) + if err != nil { + return err + } + cmd := make([]byte, cmdLen) + if _, err := io.ReadFull(r, cmd); err != nil { + return err + } + nodeID, addr, err := UnmarshalRegisterNodeAddrCmd(cmd) + if err != nil { + return err + } + s.grpcAddrs[nodeID] = addr + } + + count, err := readUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < count; i++ { + cmdLen, err := readUint32(r) + if err != nil { + return err + } + cmd := make([]byte, cmdLen) + if _, err := io.ReadFull(r, cmd); err != nil { + return err + } + topo, err := UnmarshalUpdateTopologyCmd(cmd) + if err != nil { + return err + } + s.topology.Shards[topo.ShardID] = topo + if topo.Epoch > s.topology.Epoch { + s.topology.Epoch = topo.Epoch + } + } + + return nil +} + +// Close is a no-op for the in-memory state machine. +func (s *MetadataStateMachine) Close() error { + return nil +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +func (s *MetadataStateMachine) copyTopology() *TopologySnapshot { + cp := &TopologySnapshot{ + Shards: make(map[uint64]*ShardTopology, len(s.topology.Shards)), + Epoch: s.topology.Epoch, + } + for id, shard := range s.topology.Shards { + cp.Shards[id] = copyShardTopology(shard) + } + return cp +} + +func copyShardTopology(t *ShardTopology) *ShardTopology { + cp := &ShardTopology{ + ShardID: t.ShardID, + LeaderID: t.LeaderID, + LeaderAddr: t.LeaderAddr, + Term: t.Term, + Epoch: t.Epoch, + ConfigChangeID: t.ConfigChangeID, + Nodes: make(map[uint64]string, len(t.Nodes)), + NonVotings: make(map[uint64]string, len(t.NonVotings)), + Witnesses: make(map[uint64]string, len(t.Witnesses)), + GrpcAddrs: make(map[uint64]string, len(t.GrpcAddrs)), + } + for k, v := range t.Nodes { + cp.Nodes[k] = v + } + for k, v := range t.NonVotings { + cp.NonVotings[k] = v + } + for k, v := range t.Witnesses { + cp.Witnesses[k] = v + } + for k, v := range t.GrpcAddrs { + cp.GrpcAddrs[k] = v + } + return cp +} + +func writeUint32(w io.Writer, v uint32) error { + b := []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} + _, err := w.Write(b) + return err +} + +func readUint32(r io.Reader) (uint32, error) { + b := make([]byte, 4) + if _, err := io.ReadFull(r, b); err != nil { + return 0, err + } + return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]), nil +} diff --git a/pkg/raft/metadata/statemachine_test.go b/pkg/raft/metadata/statemachine_test.go new file mode 100644 index 0000000..cff07f9 --- /dev/null +++ b/pkg/raft/metadata/statemachine_test.go @@ -0,0 +1,272 @@ +package metadata + +import ( + "bytes" + "testing" + + "github.com/lni/dragonboat/v4/statemachine" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" +) + +type StateMachineSuite struct { + suite.Suite +} + +func TestStateMachineSuite(t *testing.T) { + suite.Run(t, new(StateMachineSuite)) +} + +func newSM() *MetadataStateMachine { + factory := NewMetadataStateMachineFactory(zap.NewNop()) + sm, _ := factory(1, 1).(*MetadataStateMachine) + return sm +} + +// ─── Update: UpdateTopologyCmd ─────────────────────────────────────────────── + +func (s *StateMachineSuite) TestUpdate_UpdateTopology_StoresShard() { + require := s.Require() + + sm := newSM() + + topo := &ShardTopology{ + ShardID: 42, + LeaderID: 3, + Epoch: 10, + Nodes: map[uint64]string{1: "addr1"}, + } + cmd, err := MarshalUpdateTopologyCmd(topo) + require.NoError(err) + + result, err := sm.Update(statemachine.Entry{Cmd: cmd}) + require.NoError(err) + require.Equal(uint64(1), result.Value) +} + +func (s *StateMachineSuite) TestUpdate_UpdateTopology_EmptyCmd_Skipped() { + require := s.Require() + + sm := newSM() + + result, err := sm.Update(statemachine.Entry{Cmd: []byte{}}) + require.NoError(err) + require.Equal(uint64(0), result.Value) +} + +// ─── Update: RegisterNodeAddrCmd ───────────────────────────────────────────── + +func (s *StateMachineSuite) TestUpdate_RegisterNodeAddr_StoresAddr() { + require := s.Require() + + sm := newSM() + + cmd, err := MarshalRegisterNodeAddrCmd(5, "node5:9000") + require.NoError(err) + + result, err := sm.Update(statemachine.Entry{Cmd: cmd}) + require.NoError(err) + require.Equal(uint64(1), result.Value) + + addrs := sm.GetGrpcAddrs() + require.Equal("node5:9000", addrs[5]) +} + +// ─── Update: unknown command ────────────────────────────────────────────────── + +func (s *StateMachineSuite) TestUpdate_UnknownCommand_ReturnsZero() { + require := s.Require() + + sm := newSM() + + result, err := sm.Update(statemachine.Entry{Cmd: []byte{0xFF, 0x01, 0x02}}) + require.NoError(err) + require.Equal(uint64(0), result.Value) +} + +// ─── Lookup ────────────────────────────────────────────────────────────────── + +func (s *StateMachineSuite) TestLookup_Nil_ReturnsTopologySnapshot() { + require := s.Require() + + sm := newSM() + + // Seeds a shard. + topo := &ShardTopology{ShardID: 99, Nodes: map[uint64]string{1: "a"}} + cmd, _ := MarshalUpdateTopologyCmd(topo) + _, err := sm.Update(statemachine.Entry{Cmd: cmd}) + require.NoError(err) + + result, err := sm.Lookup(nil) + require.NoError(err) + + snap, ok := result.(*TopologySnapshot) + require.True(ok) + require.Contains(snap.Shards, uint64(99)) +} + +func (s *StateMachineSuite) TestLookup_ByShardID_ReturnsShard() { + require := s.Require() + + sm := newSM() + + topo := &ShardTopology{ShardID: 42, LeaderID: 7, Nodes: map[uint64]string{}} + cmd, _ := MarshalUpdateTopologyCmd(topo) + _, err := sm.Update(statemachine.Entry{Cmd: cmd}) + require.NoError(err) + + result, err := sm.Lookup(uint64(42)) + require.NoError(err) + + shard, ok := result.(*ShardTopology) + require.True(ok) + require.Equal(uint64(42), shard.ShardID) + require.Equal(uint64(7), shard.LeaderID) +} + +func (s *StateMachineSuite) TestLookup_ByUnknownShard_ReturnsNil() { + require := s.Require() + + sm := newSM() + + result, err := sm.Lookup(uint64(999)) + require.NoError(err) + require.Nil(result) +} + +func (s *StateMachineSuite) TestLookup_StringTopology_ReturnsSnapshot() { + require := s.Require() + + sm := newSM() + + result, err := sm.Lookup("topology") + require.NoError(err) + + snap, ok := result.(*TopologySnapshot) + require.True(ok) + require.NotNil(snap.Shards) +} + +func (s *StateMachineSuite) TestLookup_UnknownString_ReturnsNil() { + require := s.Require() + + sm := newSM() + + result, err := sm.Lookup("unknown-query") + require.NoError(err) + require.Nil(result) +} + +// ─── GetTopology / GetShardTopology / GetGrpcAddrs ───────────────────────────── + +func (s *StateMachineSuite) TestGetTopology_ReturnsCopy() { + require := s.Require() + + sm := newSM() + + topo := &ShardTopology{ShardID: 7, Epoch: 5, Nodes: map[uint64]string{1: "a"}} + cmd, _ := MarshalUpdateTopologyCmd(topo) + _, err := sm.Update(statemachine.Entry{Cmd: cmd}) + require.NoError(err) + + snap := sm.GetTopology() + require.NotNil(snap) + require.Contains(snap.Shards, uint64(7)) + + // Mutating the returned snapshot should NOT affect the SM. + delete(snap.Shards, 7) + snap2 := sm.GetTopology() + require.Contains(snap2.Shards, uint64(7)) +} + +func (s *StateMachineSuite) TestGetShardTopology_ReturnsCopy() { + require := s.Require() + + sm := newSM() + + topo := &ShardTopology{ShardID: 7, LeaderID: 2, Nodes: map[uint64]string{1: "m"}} + cmd, _ := MarshalUpdateTopologyCmd(topo) + _, err := sm.Update(statemachine.Entry{Cmd: cmd}) + require.NoError(err) + + shard := sm.GetShardTopology(7) + require.NotNil(shard) + require.Equal(uint64(2), shard.LeaderID) + + // Mutating the copy should not affect the SM. + shard.LeaderID = 99 + shard2 := sm.GetShardTopology(7) + require.Equal(uint64(2), shard2.LeaderID) +} + +func (s *StateMachineSuite) TestGetGrpcAddrs_ReturnsCopy() { + require := s.Require() + + sm := newSM() + + cmd, _ := MarshalRegisterNodeAddrCmd(1, "addr-1") + _, err := sm.Update(statemachine.Entry{Cmd: cmd}) + require.NoError(err) + + addrs := sm.GetGrpcAddrs() + require.Equal("addr-1", addrs[1]) + + // Mutating the map should not affect the SM. + delete(addrs, 1) + addrs2 := sm.GetGrpcAddrs() + require.Contains(addrs2, uint64(1)) +} + +// ─── Snapshot round-trip ────────────────────────────────────────────────────── + +func (s *StateMachineSuite) TestSnapshot_RoundTrip() { + require := s.Require() + + sm1 := newSM() + + // Seed state. + topo := &ShardTopology{ + ShardID: 42, + LeaderID: 1, + LeaderAddr: "host:9090", + Epoch: 33, + Nodes: map[uint64]string{1: "raft-1", 2: "raft-2"}, + GrpcAddrs: map[uint64]string{1: "grpc-1"}, + } + topoCmd, _ := MarshalUpdateTopologyCmd(topo) + _, err := sm1.Update(statemachine.Entry{Cmd: topoCmd}) + require.NoError(err) + + addrCmd, _ := MarshalRegisterNodeAddrCmd(1, "grpc-1") + _, err = sm1.Update(statemachine.Entry{Cmd: addrCmd}) + require.NoError(err) + + // Save snapshot. + var buf bytes.Buffer + err = sm1.SaveSnapshot(&buf, nil, nil) + require.NoError(err) + + // Recover into fresh SM. + sm2 := newSM() + err = sm2.RecoverFromSnapshot(&buf, nil, nil) + require.NoError(err) + + snap := sm2.GetTopology() + require.Contains(snap.Shards, uint64(42)) + shard := snap.Shards[42] + require.Equal(uint64(1), shard.LeaderID) + require.Equal("host:9090", shard.LeaderAddr) + require.Equal(map[uint64]string{1: "raft-1", 2: "raft-2"}, shard.Nodes) + + addrs := sm2.GetGrpcAddrs() + require.Equal("grpc-1", addrs[1]) +} + +// ─── Close ──────────────────────────────────────────────────────────────────── + +func (s *StateMachineSuite) TestClose_IsNoOp() { + require := s.Require() + + sm := newSM() + require.NoError(sm.Close()) +} diff --git a/pkg/utils/keys.go b/pkg/utils/keys.go new file mode 100644 index 0000000..66aa951 --- /dev/null +++ b/pkg/utils/keys.go @@ -0,0 +1,101 @@ +package utils + +import ( + "encoding/binary" + "fmt" + "time" + + "github.com/cespare/xxhash/v2" +) + +var ( + errInvalidKeyLength = "key length must be 24. got %d" +) + +// TopicHash computes a stable 64-bit hash of a topic name using xxhash64. +// This hash is embedded in Pebble keys to enable fast topic-based range scans +// without deserializing message values. +func TopicHash(topic string) uint64 { + return xxhash.Sum64String(topic) +} + +// CalculateBucket maps a Unix-millisecond timestamp to its bucket index. +// The bucket is the time divided by bucketSize. If bucketSize is zero, +// the raw millisecond timestamp is used as the bucket (maximum precision). +// Negative timestamps (invalid for scheduled messages) return bucket 0. +func CalculateBucket(unixMs int64, bucketSize time.Duration) uint64 { + if unixMs <= 0 { + return 0 + } + if bucketSize <= 0 { + return uint64(unixMs) + } + return uint64(unixMs) / uint64(bucketSize.Milliseconds()) +} + +// EventKey constructs the 24-byte Pebble key for a stored message. +// +// Layout (big-endian, lexicographically sortable): +// +// [0..7] topicHash uint64 — xxhash64(topic) +// [8..15] bucket uint64 — time bucket (enqueued_at_ms + delay_ms) / timeBucketSize +// [16..23] eventID uint64 — monotonic counter from EventRepository +// +// Sorting by this key groups messages by topic first, then by time bucket +// within each topic, enabling efficient per-topic range scans. +func EventKey(bucket, topicHash, eventID uint64) []byte { + key := make([]byte, 24) + binary.BigEndian.PutUint64(key[0:8], topicHash) + binary.BigEndian.PutUint64(key[8:16], bucket) + binary.BigEndian.PutUint64(key[16:24], eventID) + return key +} + +// TopicLowerBound returns the inclusive lower-bound key for scanning all +// messages belonging to a specific topic. +func TopicLowerBound(topicHash uint64) []byte { + key := make([]byte, 8) + binary.BigEndian.PutUint64(key, topicHash) + return key +} + +// TopicUpperBound returns the exclusive upper-bound key for scanning all +// messages belonging to a specific topic. +func TopicUpperBound(topicHash uint64) []byte { + key := make([]byte, 8) + binary.BigEndian.PutUint64(key, topicHash+1) + return key +} + +// DueUpperBound returns the exclusive upper-bound key for scanning only the +// due (already-scheduled) messages of a topic — those whose bucket is at most +// maxBucket. The returned 16-byte prefix is [topicHash][maxBucket+1]; since +// EventKey is [topicHash][bucket][eventID] with big-endian sortable fields, +// this bound covers exactly the keys with bucket ≤ maxBucket. +func DueUpperBound(topicHash, maxBucket uint64) []byte { + key := make([]byte, 16) + binary.BigEndian.PutUint64(key[0:8], topicHash) + binary.BigEndian.PutUint64(key[8:16], maxBucket+1) + return key +} + +// BucketUpperBound returns the exclusive upper-bound key for an iterator that +// should stop after processing all entries in buckets [0..maxBucket]. +func BucketUpperBound(maxBucket uint64) []byte { + key := make([]byte, 8) + binary.BigEndian.PutUint64(key, maxBucket+1) + return key +} + +// ParseEventKey extracts the three components of a 24-byte event key. +// Returns (topicHash, bucket, eventID) matching the byte layout [topicHash][bucket][eventID]. +// Returns ok=false if the key length is not exactly 24 bytes. +func ParseEventKey(key []byte) (topicHash, bucket, eventID uint64, err error) { + if len(key) != 24 { + return 0, 0, 0, fmt.Errorf(errInvalidKeyLength, len(key)) + } + topicHash = binary.BigEndian.Uint64(key[0:8]) + bucket = binary.BigEndian.Uint64(key[8:16]) + eventID = binary.BigEndian.Uint64(key[16:24]) + return topicHash, bucket, eventID, nil +} diff --git a/pkg/utils/keys_test.go b/pkg/utils/keys_test.go new file mode 100644 index 0000000..8008522 --- /dev/null +++ b/pkg/utils/keys_test.go @@ -0,0 +1,272 @@ +package utils + +import ( + "encoding/binary" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type UtilsSuite struct { + suite.Suite +} + +func TestUtilsSuite(t *testing.T) { + suite.Run(t, new(UtilsSuite)) +} + +// ─── TopicHash ───────────────────────────────────────────────────────────── + +func (s *UtilsSuite) TestTopicHash_Deterministic() { + require := s.Require() + + h1 := TopicHash("orders.created") + h2 := TopicHash("orders.created") + require.Equal(h1, h2, "same topic should produce the same hash") +} + +func (s *UtilsSuite) TestTopicHash_Distinct() { + require := s.Require() + + h1 := TopicHash("orders.created") + h2 := TopicHash("orders.cancelled") + require.NotEqual(h1, h2, "different topics should produce different hashes") +} + +func (s *UtilsSuite) TestTopicHash_EmptyString() { + require := s.Require() + + h := TopicHash("") + require.NotZero(h, "empty string should still produce a valid hash") +} + +// ─── CalculateBucket ─────────────────────────────────────────────────────── + +func (s *UtilsSuite) TestCalculateBucket_ExactMultiple() { + require := s.Require() + + // 17000 / 1000 = 17 + require.Equal(uint64(17), CalculateBucket(17000, 1*time.Second)) +} + +func (s *UtilsSuite) TestCalculateBucket_FloorDivision() { + require := s.Require() + + // 17999 / 1000 = 17 (floor) + require.Equal(uint64(17), CalculateBucket(17999, 1*time.Second)) + // 18000 / 1000 = 18 + require.Equal(uint64(18), CalculateBucket(18000, 1*time.Second)) +} + +func (s *UtilsSuite) TestCalculateBucket_ZeroTimestamp() { + require := s.Require() + + require.Equal(uint64(0), CalculateBucket(0, 1*time.Second)) +} + +func (s *UtilsSuite) TestCalculateBucket_NegativeTimestamp() { + require := s.Require() + + require.Equal(uint64(0), CalculateBucket(-100, 1*time.Second)) + require.Equal(uint64(0), CalculateBucket(-1, 500*time.Millisecond)) +} + +func (s *UtilsSuite) TestCalculateBucket_ZeroBucketSize() { + require := s.Require() + + // When bucketSize is 0, return raw ms as bucket + require.Equal(uint64(17300), CalculateBucket(17300, 0)) +} + +func (s *UtilsSuite) TestCalculateBucket_NegativeBucketSize() { + require := s.Require() + + // Negative bucketSize should be treated same as zero (raw ms) + require.Equal(uint64(5000), CalculateBucket(5000, -1*time.Second)) +} + +func (s *UtilsSuite) TestCalculateBucket_MillisecondPrecision() { + require := s.Require() + + require.Equal(uint64(15), CalculateBucket(15, 1*time.Millisecond)) + require.Equal(uint64(150), CalculateBucket(150, 1*time.Millisecond)) +} + +// ─── EventKey ────────────────────────────────────────────────────────────── + +func (s *UtilsSuite) TestEventKey_Length() { + require := s.Require() + + key := EventKey(1, 2, 3) + require.Len(key, 24, "EventKey must always return a 24-byte key") +} + +func (s *UtilsSuite) TestEventKey_ByteLayout() { + require := s.Require() + + bucket := uint64(0x0102030405060708) + topicHash := uint64(0x1112131415161718) + eventID := uint64(0x2122232425262728) + + key := EventKey(bucket, topicHash, eventID) + + require.Equal(topicHash, binary.BigEndian.Uint64(key[0:8]), "bytes 0-7 should be topicHash") + require.Equal(bucket, binary.BigEndian.Uint64(key[8:16]), "bytes 8-15 should be bucket") + require.Equal(eventID, binary.BigEndian.Uint64(key[16:24]), "bytes 16-23 should be eventID") +} + +func (s *UtilsSuite) TestEventKey_LexicographicOrdering() { + require := s.Require() + + // Smaller topicHash should sort before larger + key1 := EventKey(1, 100, 1) + key2 := EventKey(1, 200, 1) + require.Less(string(key1), string(key2), "keys should be lexicographically sortable by topicHash first") + + // Same topic, smaller bucket should sort before larger + key3 := EventKey(10, 100, 1) + key4 := EventKey(20, 100, 1) + require.Less(string(key3), string(key4), "within same topic, smaller bucket should come first") + + // Same topic + bucket, smaller eventID should sort first + key5 := EventKey(10, 100, 1) + key6 := EventKey(10, 100, 2) + require.Less(string(key5), string(key6), "within same bucket, smaller eventID should come first") +} + +// ─── TopicLowerBound / TopicUpperBound ───────────────────────────────────── + +func (s *UtilsSuite) TestTopicLowerBound() { + require := s.Require() + + lb := TopicLowerBound(42) + require.Len(lb, 8) + require.Equal(uint64(42), binary.BigEndian.Uint64(lb)) +} + +func (s *UtilsSuite) TestTopicUpperBound() { + require := s.Require() + + ub := TopicUpperBound(42) + require.Len(ub, 8) + require.Equal(uint64(43), binary.BigEndian.Uint64(ub)) +} + +func (s *UtilsSuite) TestTopicBounds_Ordering() { + require := s.Require() + + lb := TopicLowerBound(100) + ub := TopicUpperBound(100) + require.Less(string(lb), string(ub), "lower bound must be less than upper bound") +} + +// ─── DueUpperBound ───────────────────────────────────────────────────────── + +func (s *UtilsSuite) TestDueUpperBound_Structure() { + require := s.Require() + + ub := DueUpperBound(42, 17) + require.Len(ub, 16) + require.Equal(uint64(42), binary.BigEndian.Uint64(ub[0:8])) + require.Equal(uint64(18), binary.BigEndian.Uint64(ub[8:16])) +} + +func (s *UtilsSuite) TestDueUpperBound_CoversDueKeys() { + require := s.Require() + + topic := uint64(7) + maxBucket := uint64(10) + ub := DueUpperBound(topic, maxBucket) + + // Keys with bucket ≤ maxBucket must sort BEFORE the upper bound (i.e. inside the range). + for _, b := range []uint64{0, 1, 5, 10} { + k := EventKey(b, topic, 999) + require.Less(string(k), string(ub), "due key bucket=%d must be inside range", b) + } + + // Keys with bucket > maxBucket must sort AT OR AFTER the upper bound (i.e. outside). + for _, b := range []uint64{11, 12, 1000} { + k := EventKey(b, topic, 0) + require.GreaterOrEqual(string(k), string(ub), "future key bucket=%d must be outside range", b) + } +} + +func (s *UtilsSuite) TestDueUpperBound_ExcludesOtherTopics() { + require := s.Require() + + ub := DueUpperBound(7, 100) + + // A key for a different topic must be outside the range. + other := EventKey(0, 8, 0) // different topicHash + require.GreaterOrEqual(string(other), string(ub)) +} + +// ─── BucketUpperBound ────────────────────────────────────────────────────── + +func (s *UtilsSuite) TestBucketUpperBound() { + require := s.Require() + + ub := BucketUpperBound(17) + require.Len(ub, 8) + require.Equal(uint64(18), binary.BigEndian.Uint64(ub)) +} + +func (s *UtilsSuite) TestBucketUpperBound_Zero() { + require := s.Require() + + ub := BucketUpperBound(0) + require.Equal(uint64(1), binary.BigEndian.Uint64(ub)) +} + +// ─── ParseEventKey ───────────────────────────────────────────────────────── + +func (s *UtilsSuite) TestParseEventKey_Valid() { + require := s.Require() + + topicHash := uint64(0xCAFEBABE) + bucket := uint64(12345) + eventID := uint64(67890) + + key := EventKey(bucket, topicHash, eventID) + th, b, eid, err := ParseEventKey(key) + + require.NoError(err) + require.Equal(topicHash, th) + require.Equal(bucket, b) + require.Equal(eventID, eid) +} + +func (s *UtilsSuite) TestParseEventKey_InvalidLength() { + require := s.Require() + + tests := []struct { + name string + key []byte + }{ + {"empty", []byte{}}, + {"too short (8)", make([]byte, 8)}, + {"too short (16)", make([]byte, 16)}, + {"too short (23)", make([]byte, 23)}, + {"too long (25)", make([]byte, 25)}, + {"too long (32)", make([]byte, 32)}, + } + + for _, tt := range tests { + s.Run(tt.name, func() { + _, _, _, err := ParseEventKey(tt.key) + require.Error(err, "expected error for key length %d", len(tt.key)) + }) + } +} + +func (s *UtilsSuite) TestParseEventKey_RoundTrip() { + require := s.Require() + + original := EventKey(999, 0xDEADBEEF, 42) + th, b, eid, err := ParseEventKey(original) + require.NoError(err) + + reconstructed := EventKey(b, th, eid) + require.Equal(original, reconstructed, "round-trip must reproduce the original key") +}