Skip to content

Commit 96c8879

Browse files
simotasimota
committed
feat(redshift): add local compatibility server
1 parent 2556462 commit 96c8879

36 files changed

Lines changed: 14681 additions & 86 deletions

‎.agents/radar.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
| Date | Area | Action | Verification |
44
| --- | --- | --- | --- |
55
| 2026-05-04 | Pub/Sub tests | Added gRPC pagination, ack deadline, and ack release regression coverage. | `go test ./...`; `VERIFY_STAGE=full-compat bash scripts/pubsub-full-compat-autoloop/verify.sh`; Pub/Sub coverage 77.2%. |
6+
| 2026-05-05 | Redshift tests | Added PostgreSQL backend type/null/command-tag safety tests and Redshift translator edge coverage for function rewrites, metadata extraction, and malformed DDL. | `go test ./internal/services/redshift/... -cover`; `VERIFY_STAGE=full-postgres bash scripts/redshift-postgres-backend-autoloop/verify.sh`. |

‎.agents/voyager.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
| --- | --- | --- | --- | --- |
33
| 2026-05-02 | Voyager | added SQS API E2E smoke journey | `scripts/sqs-e2e.sh`, `scripts/sqs-autoloop/verify.sh`, `README.md` | standalone SQS E2E and autoloop full gate pass |
44
| 2026-05-04 | Voyager | expanded Pub/Sub E2E journey across REST, dashboard API, and dashboard page serving | `scripts/pubsub-e2e.sh` | standalone Pub/Sub E2E and autoloop full gate pass |
5+
| 2026-05-05 | Voyager | expanded Redshift E2E journey across SQL, Data API, management API, and dashboard Redshift APIs | `scripts/redshift-e2e.sh` | covers status, clusters, catalog, table detail, query runner, and statement history with backend config overrides |

‎internal/app/config.go‎

Lines changed: 309 additions & 19 deletions
Large diffs are not rendered by default.

‎internal/app/config_test.go‎

Lines changed: 246 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"errors"
55
"os"
66
"path/filepath"
7+
"strings"
78
"testing"
89
)
910

@@ -37,6 +38,9 @@ func TestInitWorkspaceCreatesDefaultsWithoutOverwritingConfig(t *testing.T) {
3738
if _, err := os.Stat(filepath.Join(cfg.Storage.Path, "bigquery")); err != nil {
3839
t.Fatalf("bigquery storage not created: %v", err)
3940
}
41+
if _, err := os.Stat(filepath.Join(cfg.Storage.Path, "redshift")); err != nil {
42+
t.Fatalf("redshift storage not created: %v", err)
43+
}
4044
if _, err := os.Stat(filepath.Join(cfg.Storage.Path, "sqs")); err != nil {
4145
t.Fatalf("sqs storage not created: %v", err)
4246
}
@@ -66,6 +70,134 @@ func TestInitWorkspaceCreatesDefaultsWithoutOverwritingConfig(t *testing.T) {
6670
}
6771
}
6872

73+
func TestGenerateDefaultConfigUsesRedshiftManagedPostgresBackend(t *testing.T) {
74+
chdir(t, t.TempDir())
75+
cfg := DefaultConfig()
76+
77+
if err := InitWorkspace(cfg); err != nil {
78+
t.Fatalf("InitWorkspace() error = %v", err)
79+
}
80+
81+
configPath := filepath.Join(".devcloud", "config.yaml")
82+
data, err := os.ReadFile(configPath)
83+
if err != nil {
84+
t.Fatalf("read generated config: %v", err)
85+
}
86+
generated := string(data)
87+
for _, want := range []string{
88+
"backend:\n kind: postgres\n mode: managed",
89+
"externalDsn: ",
90+
"managed: true",
91+
} {
92+
if !strings.Contains(generated, want) {
93+
t.Fatalf("generated config missing %q:\n%s", want, generated)
94+
}
95+
}
96+
97+
loaded, err := LoadConfig(configPath)
98+
if err != nil {
99+
t.Fatalf("LoadConfig(generated) error = %v", err)
100+
}
101+
if loaded.Services.Redshift.Backend.Kind != "postgres" || loaded.Services.Redshift.Backend.Mode != "managed" || loaded.Services.Redshift.Backend.ExternalDSN != "" || !loaded.Services.Redshift.Backend.Managed {
102+
t.Fatalf("generated Redshift backend = %#v", loaded.Services.Redshift.Backend)
103+
}
104+
}
105+
106+
func TestLoadConfigPreservesExplicitRedshiftMemoryFallback(t *testing.T) {
107+
dir := t.TempDir()
108+
path := filepath.Join(dir, "config.yaml")
109+
data := []byte(`project: dev
110+
111+
services:
112+
redshift:
113+
backend:
114+
kind: memory
115+
`)
116+
if err := os.WriteFile(path, data, 0o600); err != nil {
117+
t.Fatalf("write config: %v", err)
118+
}
119+
120+
cfg, err := LoadConfig(path)
121+
if err != nil {
122+
t.Fatalf("LoadConfig() error = %v", err)
123+
}
124+
if cfg.Services.Redshift.Backend.Kind != "memory" {
125+
t.Fatalf("Services.Redshift.Backend.Kind = %q, want memory", cfg.Services.Redshift.Backend.Kind)
126+
}
127+
}
128+
129+
func TestGenerateDefaultConfigPreservesExplicitRedshiftMemoryFallback(t *testing.T) {
130+
chdir(t, t.TempDir())
131+
cfg := DefaultConfig()
132+
cfg.Services.Redshift.Backend.Kind = "memory"
133+
cfg.Services.Redshift.Backend.Mode = ""
134+
cfg.Services.Redshift.Backend.ExternalDSN = ""
135+
cfg.Services.Redshift.Backend.Managed = true
136+
137+
if err := InitWorkspace(cfg); err != nil {
138+
t.Fatalf("InitWorkspace() error = %v", err)
139+
}
140+
141+
configPath := filepath.Join(".devcloud", "config.yaml")
142+
data, err := os.ReadFile(configPath)
143+
if err != nil {
144+
t.Fatalf("read generated config: %v", err)
145+
}
146+
generated := string(data)
147+
for _, want := range []string{
148+
"backend:\n kind: memory\n mode: memory",
149+
"managed: false",
150+
} {
151+
if !strings.Contains(generated, want) {
152+
t.Fatalf("generated config missing %q:\n%s", want, generated)
153+
}
154+
}
155+
156+
loaded, err := LoadConfig(configPath)
157+
if err != nil {
158+
t.Fatalf("LoadConfig(generated) error = %v", err)
159+
}
160+
if loaded.Services.Redshift.Backend.Kind != "memory" || loaded.Services.Redshift.Backend.Mode != "memory" || loaded.Services.Redshift.Backend.Managed {
161+
t.Fatalf("generated Redshift memory fallback = %#v", loaded.Services.Redshift.Backend)
162+
}
163+
}
164+
165+
func TestGenerateDefaultConfigInfersExternalRedshiftBackendMode(t *testing.T) {
166+
chdir(t, t.TempDir())
167+
cfg := DefaultConfig()
168+
cfg.Services.Redshift.Backend.Mode = ""
169+
cfg.Services.Redshift.Backend.ExternalDSN = "postgres://dev:secret@127.0.0.1:5432/dev?sslmode=disable"
170+
cfg.Services.Redshift.Backend.Managed = true
171+
172+
if err := InitWorkspace(cfg); err != nil {
173+
t.Fatalf("InitWorkspace() error = %v", err)
174+
}
175+
176+
configPath := filepath.Join(".devcloud", "config.yaml")
177+
data, err := os.ReadFile(configPath)
178+
if err != nil {
179+
t.Fatalf("read generated config: %v", err)
180+
}
181+
generated := string(data)
182+
for _, want := range []string{
183+
"backend:\n kind: postgres\n mode: external",
184+
"externalDsn: postgres://dev:secret@127.0.0.1:5432/dev?sslmode=disable",
185+
"managed: false",
186+
} {
187+
if !strings.Contains(generated, want) {
188+
t.Fatalf("generated config missing %q:\n%s", want, generated)
189+
}
190+
}
191+
192+
loaded, err := LoadConfig(configPath)
193+
if err != nil {
194+
t.Fatalf("LoadConfig(generated) error = %v", err)
195+
}
196+
if loaded.Services.Redshift.Backend.Kind != "postgres" || loaded.Services.Redshift.Backend.Mode != "external" || loaded.Services.Redshift.Backend.ExternalDSN == "" || loaded.Services.Redshift.Backend.Managed {
197+
t.Fatalf("generated Redshift external backend = %#v", loaded.Services.Redshift.Backend)
198+
}
199+
}
200+
69201
func TestLoadConfigReadsGeneratedConfigValues(t *testing.T) {
70202
dir := t.TempDir()
71203
path := filepath.Join(dir, "config.yaml")
@@ -78,6 +210,8 @@ server:
78210
gcsPort: 4444
79211
dynamodbPort: 8100
80212
bigqueryPort: 9051
213+
redshiftPort: 15439
214+
redshiftAPIPort: 19099
81215
sqsPort: 9325
82216
pubsubGrpcPort: 18085
83217
pubsubRestPort: 18086
@@ -101,6 +235,13 @@ auth:
101235
mode: bearer-dev
102236
project: custom-bigquery-project
103237
bearerToken: bigquery-token
238+
redshift:
239+
mode: strict
240+
user: analyst
241+
password: local-password
242+
accessKeyId: local-redshift
243+
secretAccessKey: redshift-secret
244+
accountId: "210987654321"
104245
sqs:
105246
mode: strict
106247
accessKeyId: local-sqs
@@ -150,6 +291,33 @@ services:
150291
maxResultRows: 500
151292
maxExecutionSeconds: 7
152293
defaultUseLegacySql: true
294+
redshift:
295+
enabled: true
296+
region: ap-northeast-1
297+
clusterIdentifier: local-cluster
298+
database: warehouse
299+
dataDir: custom-redshift
300+
nodeType: ra3.xlplus
301+
numberOfNodes: 2
302+
maxStatementBytes: 2048
303+
backend:
304+
kind: postgres
305+
mode: external
306+
externalDsn: postgres://dev:secret@127.0.0.1:5432/dev?sslmode=disable
307+
managed: false
308+
dataApi:
309+
enabled: true
310+
maxResultBytes: 4096
311+
maxResultRows: 50
312+
statementRetentionSeconds: 60
313+
sessionRetentionSeconds: 120
314+
sql:
315+
enableExtendedProtocol: true
316+
maxResultRows: 75
317+
defaultSearchPath: analytics
318+
copyUnload:
319+
enableLocalS3: false
320+
maxInputRowBytes: 1024
153321
sqs:
154322
enabled: true
155323
region: ap-northeast-1
@@ -187,7 +355,7 @@ services:
187355
if cfg.Project != "custom" {
188356
t.Fatalf("Project = %q", cfg.Project)
189357
}
190-
if cfg.Server.SMTPPort != 2525 || cfg.Server.DashboardPort != 8825 || cfg.Server.S3Port != 4567 || cfg.Server.GCSPort != 4444 || cfg.Server.DynamoDBPort != 8100 || cfg.Server.BigQueryPort != 9051 || cfg.Server.SQSPort != 9325 || cfg.Server.PubSubGRPCPort != 18085 || cfg.Server.PubSubRESTPort != 18086 {
358+
if cfg.Server.SMTPPort != 2525 || cfg.Server.DashboardPort != 8825 || cfg.Server.S3Port != 4567 || cfg.Server.GCSPort != 4444 || cfg.Server.DynamoDBPort != 8100 || cfg.Server.BigQueryPort != 9051 || cfg.Server.RedshiftPort != 15439 || cfg.Server.RedshiftAPIPort != 19099 || cfg.Server.SQSPort != 9325 || cfg.Server.PubSubGRPCPort != 18085 || cfg.Server.PubSubRESTPort != 18086 {
191359
t.Fatalf("Server = %#v", cfg.Server)
192360
}
193361
if cfg.Auth.S3.AccessKeyID != "local" || cfg.Auth.S3.SecretAccessKey != "secret" {
@@ -202,6 +370,9 @@ services:
202370
if cfg.Auth.BigQuery.Mode != "bearer-dev" || cfg.Auth.BigQuery.Project != "custom-bigquery-project" || cfg.Auth.BigQuery.BearerToken != "bigquery-token" {
203371
t.Fatalf("Auth.BigQuery = %#v", cfg.Auth.BigQuery)
204372
}
373+
if cfg.Auth.Redshift.Mode != "strict" || cfg.Auth.Redshift.User != "analyst" || cfg.Auth.Redshift.Password != "local-password" || cfg.Auth.Redshift.AccessKeyID != "local-redshift" || cfg.Auth.Redshift.SecretAccessKey != "redshift-secret" || cfg.Auth.Redshift.AccountID != "210987654321" {
374+
t.Fatalf("Auth.Redshift = %#v", cfg.Auth.Redshift)
375+
}
205376
if cfg.Auth.SQS.Mode != "strict" || cfg.Auth.SQS.AccessKeyID != "local-sqs" || cfg.Auth.SQS.SecretAccessKey != "sqs-secret" || cfg.Auth.SQS.AccountID != "123456789012" {
206377
t.Fatalf("Auth.SQS = %#v", cfg.Auth.SQS)
207378
}
@@ -238,6 +409,24 @@ services:
238409
if cfg.Services.BigQuery.Query.MaxResultRows != 500 || cfg.Services.BigQuery.Query.MaxExecutionSeconds != 7 || !cfg.Services.BigQuery.Query.DefaultUseLegacySQL {
239410
t.Fatalf("Services.BigQuery.Query = %#v", cfg.Services.BigQuery.Query)
240411
}
412+
if !cfg.Services.Redshift.Enabled || cfg.Services.Redshift.Region != "ap-northeast-1" || cfg.Services.Redshift.ClusterIdentifier != "local-cluster" || cfg.Services.Redshift.Database != "warehouse" {
413+
t.Fatalf("Services.Redshift = %#v", cfg.Services.Redshift)
414+
}
415+
if cfg.Services.Redshift.DataDir != "custom-redshift" || cfg.Services.Redshift.NodeType != "ra3.xlplus" || cfg.Services.Redshift.NumberOfNodes != 2 || cfg.Services.Redshift.MaxStatementBytes != 2048 {
416+
t.Fatalf("Services.Redshift metadata = %#v", cfg.Services.Redshift)
417+
}
418+
if cfg.Services.Redshift.Backend.Kind != "postgres" || cfg.Services.Redshift.Backend.Mode != "external" || cfg.Services.Redshift.Backend.ExternalDSN != "postgres://dev:secret@127.0.0.1:5432/dev?sslmode=disable" || cfg.Services.Redshift.Backend.Managed {
419+
t.Fatalf("Services.Redshift.Backend = %#v", cfg.Services.Redshift.Backend)
420+
}
421+
if !cfg.Services.Redshift.DataAPI.Enabled || cfg.Services.Redshift.DataAPI.MaxResultBytes != 4096 || cfg.Services.Redshift.DataAPI.MaxResultRows != 50 || cfg.Services.Redshift.DataAPI.StatementRetentionSeconds != 60 || cfg.Services.Redshift.DataAPI.SessionRetentionSeconds != 120 {
422+
t.Fatalf("Services.Redshift.DataAPI = %#v", cfg.Services.Redshift.DataAPI)
423+
}
424+
if !cfg.Services.Redshift.SQL.EnableExtendedProtocol || cfg.Services.Redshift.SQL.MaxResultRows != 75 || cfg.Services.Redshift.SQL.DefaultSearchPath != "analytics" {
425+
t.Fatalf("Services.Redshift.SQL = %#v", cfg.Services.Redshift.SQL)
426+
}
427+
if cfg.Services.Redshift.CopyUnload.EnableLocalS3 || cfg.Services.Redshift.CopyUnload.MaxInputRowBytes != 1024 {
428+
t.Fatalf("Services.Redshift.CopyUnload = %#v", cfg.Services.Redshift.CopyUnload)
429+
}
241430
if !cfg.Services.SQS.Enabled || cfg.Services.SQS.Region != "ap-northeast-1" || cfg.Services.SQS.QueueURLHost != "localhost" {
242431
t.Fatalf("Services.SQS = %#v", cfg.Services.SQS)
243432
}
@@ -291,6 +480,12 @@ func TestDefaultConfigIncludesS3GCSAndDynamoDBDefaults(t *testing.T) {
291480
if cfg.Server.BigQueryPort != 9050 {
292481
t.Fatalf("Server.BigQueryPort = %d", cfg.Server.BigQueryPort)
293482
}
483+
if cfg.Server.RedshiftPort != 5439 {
484+
t.Fatalf("Server.RedshiftPort = %d", cfg.Server.RedshiftPort)
485+
}
486+
if cfg.Server.RedshiftAPIPort != 9099 {
487+
t.Fatalf("Server.RedshiftAPIPort = %d", cfg.Server.RedshiftAPIPort)
488+
}
294489
if cfg.Server.SQSPort != 9324 {
295490
t.Fatalf("Server.SQSPort = %d", cfg.Server.SQSPort)
296491
}
@@ -342,6 +537,27 @@ func TestDefaultConfigIncludesS3GCSAndDynamoDBDefaults(t *testing.T) {
342537
if cfg.Services.BigQuery.Query.MaxResultRows != 10000 || cfg.Services.BigQuery.Query.MaxExecutionSeconds != 30 || cfg.Services.BigQuery.Query.DefaultUseLegacySQL {
343538
t.Fatalf("Services.BigQuery.Query = %#v", cfg.Services.BigQuery.Query)
344539
}
540+
if cfg.Auth.Redshift.Mode != "relaxed" || cfg.Auth.Redshift.User != "dev" || cfg.Auth.Redshift.Password != "dev" || cfg.Auth.Redshift.AccessKeyID != "dev" || cfg.Auth.Redshift.SecretAccessKey != "dev" || cfg.Auth.Redshift.AccountID != "000000000000" {
541+
t.Fatalf("Auth.Redshift = %#v", cfg.Auth.Redshift)
542+
}
543+
if !cfg.Services.Redshift.Enabled || cfg.Services.Redshift.Region != "us-east-1" || cfg.Services.Redshift.ClusterIdentifier != "devcloud" || cfg.Services.Redshift.Database != "dev" {
544+
t.Fatalf("Services.Redshift = %#v", cfg.Services.Redshift)
545+
}
546+
if cfg.Services.Redshift.DataDir != "redshift" || cfg.Services.Redshift.NodeType != "dc2.large" || cfg.Services.Redshift.NumberOfNodes != 1 || cfg.Services.Redshift.MaxStatementBytes != 16*1024*1024 {
547+
t.Fatalf("Services.Redshift metadata = %#v", cfg.Services.Redshift)
548+
}
549+
if cfg.Services.Redshift.Backend.Kind != "postgres" || cfg.Services.Redshift.Backend.Mode != "managed" || cfg.Services.Redshift.Backend.ExternalDSN != "" || !cfg.Services.Redshift.Backend.Managed {
550+
t.Fatalf("Services.Redshift.Backend = %#v", cfg.Services.Redshift.Backend)
551+
}
552+
if !cfg.Services.Redshift.DataAPI.Enabled || cfg.Services.Redshift.DataAPI.MaxResultBytes != 500*1024*1024 || cfg.Services.Redshift.DataAPI.MaxResultRows != 10000 || cfg.Services.Redshift.DataAPI.StatementRetentionSeconds != 86400 || cfg.Services.Redshift.DataAPI.SessionRetentionSeconds != 86400 {
553+
t.Fatalf("Services.Redshift.DataAPI = %#v", cfg.Services.Redshift.DataAPI)
554+
}
555+
if cfg.Services.Redshift.SQL.EnableExtendedProtocol || cfg.Services.Redshift.SQL.MaxResultRows != 10000 || cfg.Services.Redshift.SQL.DefaultSearchPath != "public" {
556+
t.Fatalf("Services.Redshift.SQL = %#v", cfg.Services.Redshift.SQL)
557+
}
558+
if !cfg.Services.Redshift.CopyUnload.EnableLocalS3 || cfg.Services.Redshift.CopyUnload.MaxInputRowBytes != 4*1024*1024 {
559+
t.Fatalf("Services.Redshift.CopyUnload = %#v", cfg.Services.Redshift.CopyUnload)
560+
}
345561
if cfg.Auth.SQS.Mode != "relaxed" || cfg.Auth.SQS.AccessKeyID != "dev" || cfg.Auth.SQS.SecretAccessKey != "dev" || cfg.Auth.SQS.AccountID != "000000000000" {
346562
t.Fatalf("Auth.SQS = %#v", cfg.Auth.SQS)
347563
}
@@ -424,6 +640,9 @@ func TestWorkspaceStoragePathMustStayUnderDevcloud(t *testing.T) {
424640
if _, err := os.Stat(filepath.Join(".devcloud", "custom-data", "bigquery")); err != nil {
425641
t.Fatalf("custom bigquery storage not created: %v", err)
426642
}
643+
if _, err := os.Stat(filepath.Join(".devcloud", "custom-data", "redshift")); err != nil {
644+
t.Fatalf("custom redshift storage not created: %v", err)
645+
}
427646
if _, err := os.Stat(filepath.Join(".devcloud", "custom-data", "pubsub")); err != nil {
428647
t.Fatalf("custom pubsub storage not created: %v", err)
429648
}
@@ -462,6 +681,32 @@ func TestInitWorkspaceUsesPubSubSpecificDataDirs(t *testing.T) {
462681
}
463682
}
464683

684+
func TestInitWorkspaceUsesRedshiftDataDirUnderDevcloud(t *testing.T) {
685+
chdir(t, t.TempDir())
686+
cfg := DefaultConfig()
687+
cfg.Services.Redshift.DataDir = filepath.Join(".devcloud", "redshift-store")
688+
689+
if err := InitWorkspace(cfg); err != nil {
690+
t.Fatalf("InitWorkspace() error = %v", err)
691+
}
692+
if _, err := os.Stat(cfg.Services.Redshift.DataDir); err != nil {
693+
t.Fatalf("custom redshift dataDir not created: %v", err)
694+
}
695+
696+
cfg.Services.Redshift.DataDir = "redshift-outside"
697+
if err := InitWorkspace(cfg); err != nil {
698+
t.Fatalf("relative redshift dataDir should stay under storage path: %v", err)
699+
}
700+
if _, err := os.Stat(filepath.Join(cfg.Storage.Path, "redshift-outside")); err != nil {
701+
t.Fatalf("relative redshift dataDir not created under storage path: %v", err)
702+
}
703+
704+
cfg.Services.Redshift.DataDir = filepath.Join("..", "..", "redshift-outside")
705+
if err := InitWorkspace(cfg); err == nil {
706+
t.Fatal("InitWorkspace() error = nil for redshift dataDir outside .devcloud")
707+
}
708+
}
709+
465710
func TestResetWorkspaceRemovesPubSubSpecificDataDirs(t *testing.T) {
466711
chdir(t, t.TempDir())
467712
cfg := DefaultConfig()

0 commit comments

Comments
 (0)