From 566e291f6c1aefa03f9e851432300686b5b0335c Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:59:48 -0700 Subject: [PATCH 01/12] fix(config,cli): compose import preserves web container port, refuses independent builds (C05) mapCompose used the web service's ports only for candidacy and discarded them ('8080:3000' imported as Port=0, deployed as :80); ports now resolve through composeAppPort over the documented short-form grammar with refusal for ranges, long-form objects, multiple container ports and UDP-only services, and non-TCP entries preserved into publish. A service built from a different context than web's is refused naming it (it used to flatten into a same-image process running the wrong code). Both deploy.Config literals extracted into one deployConfigFromApp so the config->deploy seam is a single tested mapping; the wiring guard now covers it. Evidence: _internal/evals/2026-09-21/implementation-slices-2026-09-21.md --- AUDIT_OPEN.md | 39 ++++ internal/cli/deploy.go | 35 +--- internal/cli/deployconfig.go | 50 +++++ internal/cli/deployconfig_test.go | 32 ++++ internal/cli/deployconfig_wiring_test.go | 12 +- internal/cli/singledeploy.go | 35 +--- internal/config/compose.go | 109 ++++++++++- internal/config/compose_test.go | 234 ++++++++++++++++++++++- 8 files changed, 464 insertions(+), 82 deletions(-) create mode 100644 internal/cli/deployconfig.go create mode 100644 internal/cli/deployconfig_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index a99ccd2..20113da 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -769,3 +769,42 @@ all packages ok. No push performed. (Environmental note: Apple's Xcode 27 update landed mid-session and required license re-acceptance for /usr/bin/git; the closing gates ran against the standalone Command Line Tools git on PATH.) + +## Product programme slices (2026-09-21) — Compose import contracts + +Two C05 findings from the product evaluation's strengthened contract +probes (`_internal/evals/2026-09-21/`), fixed as bounded slices with +evidence. Base revision `8486355`; changes left uncommitted for review. + +- **Compose port preservation** — `mapCompose` used the web service's + ports only for candidacy and silently discarded them, so + `ports: ['8080:3000']` imported with `Port=0` (deployed as `:80`, + health check probing the wrong port). Ports now resolve through + `composeAppPort` over the same narrow grammar as `ParsePublishSpec` + (short strings or bare numbers; ranges and long-form objects refused + naming the service; multiple distinct container ports refused as + ambiguous; non-TCP entries preserved verbatim into `publish`). The + Compose host-side binding is deliberately not preserved — teploy + allocates host ports and routes via Caddy. The config→deploy hop was + made observable by extracting `deployConfigFromApp` (both entry + points now share one literal, covered by the strengthened + `TestDeployConfigCopiesEveryMatchingAppConfigField` wiring guard, + which previously saw only deploy.go's copy). +- **Independent-build refusal** — a service built from a different + context than web's with no image was flattened into a same-image + process (`jobs: build ./jobs` ran web's image under the jobs + command — wrong code, right command, success reported). The import + now refuses, deterministically naming every offending service and + its build context, with the remediation (same context / prebuilt + image / teploy.yml). Full multi-image build identity remains C05. + +Gates: `go vet ./...` clean; `go test ./... -race` all packages ok; +strengthened probes — port and worker contracts PASS, both controls +PASS, the preview branch-identity probe remains a known failure (C06, +separate task). Mutation checks in a scratch copy: removing the port +assignment, substituting the host port, breaking the seam mapping, and +restoring the lossy flatten each fail the new regressions for the +intended reason. Real Docker port behavior remains a later journey +gate (J05); no remote deployment was performed. The preview collision +and full Compose breadth stay open under the product programme +(`_internal/TEPLOY_PRODUCT_EXCELLENCE_PROGRAMME_2026-09-21.md` C05/C06). diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index 0609c8d..32e964a 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -639,40 +639,7 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * // 11. Deploy. deployer := deploy.NewDeployer(executor, os.Stdout) - deployCfg := deploy.Config{ - App: appCfg.App, - Domain: appCfg.Domain, - Image: image, - Version: version, - EnvFiles: envFiles, - Volumes: volumes, - Processes: appCfg.Processes, - NoHealthcheck: disabledHealthchecks(appCfg.Healthcheck), - Health: healthConfigFrom(appCfg.Health), - KeepVersions: appCfg.KeepVersions, - Ingress: appCfg.Ingress, - Bind: appCfg.Bind, - ContainerPort: appCfg.Port, - Publish: appCfg.Publish, - StopTimeout: appCfg.StopTimeout, - Memory: appCfg.Memory, - CPU: appCfg.CPU, - Replicas: appCfg.Replicas, - PreDeploy: appCfg.Hooks.PreDeploy, - PostDeploy: appCfg.Hooks.PostDeploy, - AssetPath: appCfg.Assets.Path, - AssetKeepDays: appCfg.Assets.KeepDays, - TLSCert: tlsCert, - TLSKey: tlsKey, - TLSInternal: tlsInternal, - CaddyExtra: appCfg.CaddyExtra, - Cache: appCfg.Cache, - Firewall: caddyFirewall(appCfg.Firewall), - Access: caddyAccess(appCfg.Access), - ManifestSHA256: manifestSHA256, - AppliedManifest: appliedManifest, - SourceRevision: appCfg.SourceRevision, - } + deployCfg := deployConfigFromApp(appCfg, image, version, envFiles, volumes, tlsCert, tlsKey, tlsInternal, appliedManifest, manifestSHA256) // Vulnerability gate: scan the image on the server before any container // starts — fixable CRITICALs block the deploy. diff --git a/internal/cli/deployconfig.go b/internal/cli/deployconfig.go new file mode 100644 index 0000000..9b71520 --- /dev/null +++ b/internal/cli/deployconfig.go @@ -0,0 +1,50 @@ +package cli + +import ( + "encoding/json" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" +) + +// deployConfigFromApp maps a loaded AppConfig onto the deploy engine's +// Config. Both deploy entry points (single-server and multi-server) +// build their Config here so the config→deploy seam is one tested +// mapping: an imported application port (AppConfig.Port) reaches +// deploy.Config.ContainerPort exactly once, observably. +func deployConfigFromApp(appCfg *config.AppConfig, image, version string, envFiles []string, volumes map[string]string, tlsCert, tlsKey string, tlsInternal bool, appliedManifest json.RawMessage, manifestSHA256 string) deploy.Config { + return deploy.Config{ + App: appCfg.App, + Domain: appCfg.Domain, + Image: image, + Version: version, + EnvFiles: envFiles, + Volumes: volumes, + Processes: appCfg.Processes, + NoHealthcheck: disabledHealthchecks(appCfg.Healthcheck), + Health: healthConfigFrom(appCfg.Health), + KeepVersions: appCfg.KeepVersions, + Ingress: appCfg.Ingress, + Bind: appCfg.Bind, + ContainerPort: appCfg.Port, + Publish: appCfg.Publish, + StopTimeout: appCfg.StopTimeout, + Memory: appCfg.Memory, + CPU: appCfg.CPU, + Replicas: appCfg.Replicas, + PreDeploy: appCfg.Hooks.PreDeploy, + PostDeploy: appCfg.Hooks.PostDeploy, + AssetPath: appCfg.Assets.Path, + AssetKeepDays: appCfg.Assets.KeepDays, + TLSCert: tlsCert, + TLSKey: tlsKey, + TLSInternal: tlsInternal, + CaddyExtra: appCfg.CaddyExtra, + Cache: appCfg.Cache, + Firewall: caddyFirewall(appCfg.Firewall), + Access: caddyAccess(appCfg.Access), + ManifestSHA256: manifestSHA256, + AppliedManifest: appliedManifest, + SourceRevision: appCfg.SourceRevision, + } +} diff --git a/internal/cli/deployconfig_test.go b/internal/cli/deployconfig_test.go new file mode 100644 index 0000000..53b2200 --- /dev/null +++ b/internal/cli/deployconfig_test.go @@ -0,0 +1,32 @@ +package cli + +import ( + "testing" + + "github.com/useteploy/teploy/internal/config" +) + +// TestDeployConfigFromAppPropagatesContainerPort is the positive +// config→deploy seam check for Compose port preservation (C05): the +// application port declared in the imported config must arrive in the +// deploy engine's ContainerPort unchanged. This hop is what stayed +// unobservable while the Compose importer dropped the port entirely. +func TestDeployConfigFromAppPropagatesContainerPort(t *testing.T) { + appCfg := &config.AppConfig{App: "app", Domain: "app.example.com", Port: 3000} + got := deployConfigFromApp(appCfg, "example/web:v1", "v1", nil, nil, "", "", false, nil, "") + if got.ContainerPort != 3000 { + t.Errorf("deploy.Config.ContainerPort = %d, want 3000 (the imported application port)", got.ContainerPort) + } + if got.Image != "example/web:v1" { + t.Errorf("deploy.Config.Image = %q, want example/web:v1", got.Image) + } + + // A config with no declared port (a teploy.yml app relying on the + // default) must flow through as 0 — normalization to 80 belongs to + // the deploy engine, not this mapping. + appCfg.Port = 0 + got = deployConfigFromApp(appCfg, "example/web:v1", "v1", nil, nil, "", "", false, nil, "") + if got.ContainerPort != 0 { + t.Errorf("deploy.Config.ContainerPort = %d, want 0 (deploy normalizes the default)", got.ContainerPort) + } +} diff --git a/internal/cli/deployconfig_wiring_test.go b/internal/cli/deployconfig_wiring_test.go index f5e56dc..e7af31a 100644 --- a/internal/cli/deployconfig_wiring_test.go +++ b/internal/cli/deployconfig_wiring_test.go @@ -38,15 +38,19 @@ func TestDeployConfigCopiesEveryMatchingAppConfigField(t *testing.T) { "Env": "folded into EnvFiles by buildContainerEnvFiles, so secrets never reach the docker run argv", } - source, err := os.ReadFile("deploy.go") + // The mapping lives in one place — deployConfigFromApp in + // deployconfig.go — shared by BOTH deploy entry points (deploy.go + // and singledeploy.go), so guarding this one literal guards the + // mapping each entry point actually uses. + source, err := os.ReadFile("deployconfig.go") if err != nil { - t.Fatalf("reading deploy.go: %v", err) + t.Fatalf("reading deployconfig.go: %v", err) } // Just the deploy.Config literal, so an unrelated mention elsewhere in the // file cannot make a missing assignment look present. - literal := regexp.MustCompile(`(?s)deployCfg := deploy\.Config\{(.*?)\n\t\}`).FindSubmatch(source) + literal := regexp.MustCompile(`(?s)return deploy\.Config\{(.*?)\n\t\}`).FindSubmatch(source) if literal == nil { - t.Fatal("could not find the `deployCfg := deploy.Config{...}` literal in deploy.go") + t.Fatal("could not find the `return deploy.Config{...}` literal in deployconfig.go") } // Strip line comments before matching. `strings.Contains(body, "Memory:")` // is otherwise satisfied by `// Memory: appCfg.Memory,` — verified: deleting diff --git a/internal/cli/singledeploy.go b/internal/cli/singledeploy.go index b71cd8b..30a2690 100644 --- a/internal/cli/singledeploy.go +++ b/internal/cli/singledeploy.go @@ -265,40 +265,7 @@ func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.App return fmt.Errorf("normalizing applied manifest: %w", err) } deployer := deploy.NewDeployer(s.exec, s.out) - deployCfg := deploy.Config{ - App: appCfg.App, - Domain: appCfg.Domain, - Image: image, - Version: version, - EnvFiles: envFiles, - Volumes: volumes, - Processes: appCfg.Processes, - NoHealthcheck: disabledHealthchecks(appCfg.Healthcheck), - Health: healthConfigFrom(appCfg.Health), - KeepVersions: appCfg.KeepVersions, - Ingress: appCfg.Ingress, - Bind: appCfg.Bind, - ContainerPort: appCfg.Port, - Publish: appCfg.Publish, - StopTimeout: appCfg.StopTimeout, - Memory: appCfg.Memory, - CPU: appCfg.CPU, - Replicas: appCfg.Replicas, - PreDeploy: appCfg.Hooks.PreDeploy, - PostDeploy: appCfg.Hooks.PostDeploy, - AssetPath: appCfg.Assets.Path, - AssetKeepDays: appCfg.Assets.KeepDays, - TLSCert: tlsCert, - TLSKey: tlsKey, - TLSInternal: tlsInternal, - CaddyExtra: appCfg.CaddyExtra, - Cache: appCfg.Cache, - Firewall: caddyFirewall(appCfg.Firewall), - Access: caddyAccess(appCfg.Access), - ManifestSHA256: manifestSHA256, - AppliedManifest: appliedManifest, - SourceRevision: appCfg.SourceRevision, - } + deployCfg := deployConfigFromApp(appCfg, image, version, envFiles, volumes, tlsCert, tlsKey, tlsInternal, appliedManifest, manifestSHA256) // Vulnerability gate (see deploy.go): fixable CRITICALs block before // containers start. Per-server, so every box in a multi-server deploy diff --git a/internal/config/compose.go b/internal/config/compose.go index 694cf78..249af66 100644 --- a/internal/config/compose.go +++ b/internal/config/compose.go @@ -6,6 +6,7 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" "github.com/useteploy/teploy/internal/ssh" @@ -18,13 +19,13 @@ type composeFile struct { } type composeService struct { - Image string `yaml:"image"` - Build interface{} `yaml:"build"` // string or struct - Ports []string `yaml:"ports"` - Command interface{} `yaml:"command"` // string or []string - Environment interface{} `yaml:"environment"` // map or list - Volumes []string `yaml:"volumes"` - DependsOn interface{} `yaml:"depends_on"` // list or map + Image string `yaml:"image"` + Build interface{} `yaml:"build"` // string or struct + Ports []interface{} `yaml:"ports"` // short-form strings or bare numbers; anything else is refused in composeAppPort + Command interface{} `yaml:"command"` // string or []string + Environment interface{} `yaml:"environment"` // map or list + Volumes []string `yaml:"volumes"` + DependsOn interface{} `yaml:"depends_on"` // list or map } // knownAccessoryImages maps image prefixes to default ports. @@ -124,6 +125,20 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { } return nil, fmt.Errorf("no service with ports found in compose file") } + // The web service's ports decide the application container port. + // Compose host-side bindings are deliberately not preserved (teploy + // allocates host ports itself and routes via Caddy), while non-TCP + // publishes carry into Publish verbatim. Unsupported grammar — + // ranges, long-form entries, multiple distinct container ports — + // is refused with the reason instead of given arbitrary meaning + // (previously every port entry was used only for web candidacy and + // silently discarded, so '8080:3000' imported with Port=0 and + // deployed as :80). + webPort, extraPublish, err := composeAppPort(webServiceName, webService.Ports) + if err != nil { + return nil, err + } + webBuildContext := parseBuildContext(webService.Build) // Set domain placeholder — user must set this. @@ -137,7 +152,14 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { cfg.Image = webService.Image } + // The application port resolved from the web service's ports. + cfg.Port = webPort + if len(extraPublish) > 0 { + cfg.Publish = extraPublish + } + // Classify remaining services. + var unsupportedBuilds []string for name, svc := range compose.Services { if name == webServiceName { continue @@ -194,8 +216,22 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { continue } - // Has build context different from web → worker (different build). - cfg.Processes[name] = parseCommand(svc.Command) + // Build context different from web's with no image: the + // single-image process model cannot preserve an independent + // build. Collect and refuse below — flattening the service + // into a worker of web's image used to deploy the wrong code + // under the right command (`jobs: build ./jobs` ran web's + // image). + unsupportedBuilds = append(unsupportedBuilds, fmt.Sprintf("%s (build %q)", name, svcBuildContext)) + } + + if len(unsupportedBuilds) > 0 { + sort.Strings(unsupportedBuilds) + webSrc := fmt.Sprintf("%q builds from %q", webServiceName, webBuildContext) + if webBuildContext == "" { + webSrc = fmt.Sprintf("%q runs image %q", webServiceName, webService.Image) + } + return nil, fmt.Errorf("unsupported independent build in compose import: %s while %s — teploy runs one image per app and cannot preserve a separately built service; use the same build context as the app, a prebuilt image, or write teploy.yml", strings.Join(unsupportedBuilds, ", "), webSrc) } // Clean up empty maps. @@ -209,6 +245,61 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { return cfg, nil } +// composeAppPort resolves the web service's application container port +// from its Compose ports entries, returning the port and any non-TCP +// entries that must be preserved verbatim as publishes. The supported +// grammar is the same narrow grammar as ParsePublishSpec: short-form +// "[host:]container[/proto]" strings (or bare numbers) with single +// numeric ports. Long-form ports objects, ranges and multiple distinct +// container ports are refused naming the reason — never given arbitrary +// meaning. The Compose HOST-side binding is not part of the returned +// value: it is host plumbing that teploy replaces with its own +// allocation and Caddy routing. +func composeAppPort(service string, raw []interface{}) (int, []string, error) { + seen := map[int]bool{} + var extra []string + for _, entry := range raw { + var s string + switch v := entry.(type) { + case string: + s = v + case int: + s = strconv.Itoa(v) + default: + return 0, nil, fmt.Errorf("compose service %q ports: unsupported ports entry %v — only short \"[host:]container[/proto]\" strings or bare numbers import; write teploy.yml for long-form Compose ports", service, entry) + } + spec, err := ParsePublishSpec(s) + if err != nil { + return 0, nil, fmt.Errorf("compose service %q ports: %w", service, err) + } + if spec.Proto != "" && spec.Proto != "tcp" { + extra = append(extra, s) + continue + } + seen[spec.ContainerPort] = true + } + ports := make([]int, 0, len(seen)) + for p := range seen { + ports = append(ports, p) + } + sort.Ints(ports) + switch len(ports) { + case 1: + return ports[0], extra, nil + case 0: + if len(extra) > 0 { + return 0, nil, fmt.Errorf("compose service %q publishes only non-TCP ports — teploy serves HTTP over TCP and cannot pick an application port; write teploy.yml", service) + } + return 0, nil, fmt.Errorf("compose service %q publishes no usable TCP application port", service) + default: + var names []string + for _, p := range ports { + names = append(names, strconv.Itoa(p)) + } + return 0, nil, fmt.Errorf("ambiguous compose import: service %q publishes multiple container ports (%s) — teploy routes one application port; remove the extra ports or write teploy.yml", service, strings.Join(names, ", ")) + } +} + func parseBuildContext(build interface{}) string { switch v := build.(type) { case string: diff --git a/internal/config/compose_test.go b/internal/config/compose_test.go index 8f1f48d..afc8272 100644 --- a/internal/config/compose_test.go +++ b/internal/config/compose_test.go @@ -228,7 +228,7 @@ func TestIsAccessoryImage(t *testing.T) { {"mongo:latest", true}, {"myapp:latest", false}, {"ghcr.io/myorg/myapp:v1", false}, - {"", false}, // Registry ports: the colon before the last slash is a registry + {"", false}, // Registry ports: the colon before the last slash is a registry // host port, not a tag separator (teploy-cli-08 twin — this used // to reduce to "registry.example" and disable classification). {"registry.example:5000/postgres:16", true}, @@ -433,3 +433,235 @@ services: t.Errorf("error should mention the accessory situation, got: %v", err) } } + +// TestLoadCompose_PreservesWebContainerPort: the import contract from the +// product evaluation (C05) — a supported one-image short-port Compose file +// must import with its declared INTERNAL web port. "8080:3000" means +// container 3000 bound to host 8080 in Compose; the container port is the +// application port, and the host binding is Compose host plumbing teploy +// does not preserve. This used to import with Port=0 (deployed as :80). +func TestLoadCompose_PreservesWebContainerPort(t *testing.T) { + compose := ` +services: + web: + image: example/web:v1 + ports: ["8080:3000"] +` + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(compose), 0644) + + cfg, err := LoadCompose(dir) + if err != nil { + t.Fatalf("supported short-port Compose fixture must import: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if cfg.Port != 3000 { + t.Errorf("web container port = %d, want 3000 (the host binding 8080 is not the application port)", cfg.Port) + } + if cfg.Image != "example/web:v1" { + t.Errorf("web image = %q, want example/web:v1", cfg.Image) + } +} + +// TestLoadCompose_PortShortForms covers the supported short-form port +// grammar: host:container, bare container port (quoted and unquoted), +// IPv4- and bracketed-IPv6-prefixed bindings, and the same container port +// published through several bindings — one application port, not two. +func TestLoadCompose_PortShortForms(t *testing.T) { + tests := []struct { + name string + ports string + want int + }{ + {"host:container", `["8080:3000"]`, 3000}, + {"bare container port", `["3000"]`, 3000}, + {"bare unquoted number", "[3000]", 3000}, + {"ipv4-prefixed", `["127.0.0.1:8080:3000"]`, 3000}, + {"ipv6-prefixed", `["[::1]:8080:3000"]`, 3000}, + {"same container port twice", `["8080:3000", "127.0.0.1:8081:3000"]`, 3000}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + compose := "services:\n web:\n image: example/web:v1\n ports: " + tt.ports + "\n" + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(compose), 0644) + + cfg, err := LoadCompose(dir) + if err != nil { + t.Fatalf("supported short form must import: %v", err) + } + if cfg.Port != tt.want { + t.Errorf("application port = %d, want %d", cfg.Port, tt.want) + } + }) + } +} + +// TestLoadCompose_MultipleAppPortsRefused: two distinct container ports on +// the web service have no principled single application port — the import +// must refuse and name both rather than pick one. +func TestLoadCompose_MultipleAppPortsRefused(t *testing.T) { + compose := ` +services: + web: + image: myapp:latest + ports: ["8080:3000", "8081:3001"] +` + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(compose), 0644) + + _, err := LoadCompose(dir) + if err == nil { + t.Fatal("expected an ambiguity error for multiple container ports") + } + if !strings.Contains(err.Error(), "ambiguous") || + !strings.Contains(err.Error(), "3000") || + !strings.Contains(err.Error(), "3001") { + t.Errorf("error must name the ambiguity and both ports, got: %v", err) + } +} + +// TestLoadCompose_NonTCPPorts: a non-TCP publish is not an application +// port (teploy serves HTTP over TCP) but is preserved verbatim as a +// publish; a service with ONLY non-TCP ports is refused with the reason. +func TestLoadCompose_NonTCPPorts(t *testing.T) { + mixed := ` +services: + web: + image: example/dns:v1 + ports: ["8080:3000", "53:53/udp"] +` + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(mixed), 0644) + + cfg, err := LoadCompose(dir) + if err != nil { + t.Fatalf("mixed TCP + UDP service must import: %v", err) + } + if cfg.Port != 3000 { + t.Errorf("application port = %d, want 3000 (the TCP port)", cfg.Port) + } + if len(cfg.Publish) != 1 || cfg.Publish[0] != "53:53/udp" { + t.Errorf("UDP entry must be preserved verbatim in publish, got %v", cfg.Publish) + } + + udpOnly := ` +services: + web: + image: example/dns:v1 + ports: ["53:53/udp"] +` + dir = t.TempDir() + os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(udpOnly), 0644) + + _, err = LoadCompose(dir) + if err == nil { + t.Fatal("expected an error when only a UDP port is published") + } + if !strings.Contains(err.Error(), "TCP") { + t.Errorf("error must explain the TCP requirement, got: %v", err) + } +} + +// TestLoadCompose_PortRangeRefused: port ranges are outside the supported +// grammar and must be refused at import with the reason, before any +// deploy effect. +func TestLoadCompose_PortRangeRefused(t *testing.T) { + compose := ` +services: + web: + image: myapp:latest + ports: ["3000-3005:3000-3005"] +` + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(compose), 0644) + + _, err := LoadCompose(dir) + if err == nil { + t.Fatal("expected an error for a port range") + } + if !strings.Contains(err.Error(), "web") || !strings.Contains(err.Error(), "3000-3005") { + t.Errorf("error must name the service and the offending entry, got: %v", err) + } +} + +// TestLoadCompose_LongFormPortsRefused: Compose long-form ports objects +// (target/published maps) are outside the supported grammar — refuse +// naming the service and the supported alternative. +func TestLoadCompose_LongFormPortsRefused(t *testing.T) { + compose := ` +services: + web: + image: myapp:latest + ports: + - target: 3000 + published: 8080 +` + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(compose), 0644) + + _, err := LoadCompose(dir) + if err == nil { + t.Fatal("expected an error for long-form ports") + } + if !strings.Contains(err.Error(), "web") || !strings.Contains(err.Error(), "ports") { + t.Errorf("error must name the service and its ports, got: %v", err) + } +} + +// TestLoadCompose_RefusesIndependentBuild: a service built from a context +// different from the web service's cannot be preserved by the single-image +// process model — the import must refuse naming the service and its +// build, never silently flatten it into a worker of the app's image +// (which deployed the wrong code under the right command). +func TestLoadCompose_RefusesIndependentBuild(t *testing.T) { + tests := []struct { + name string + compose string + }{ + { + "build-based web", + ` +services: + web: + build: ./web + ports: ["3000:3000"] + jobs: + build: ./jobs + command: python jobs.py +`, + }, + { + "image-based web", + ` +services: + web: + image: myapp:latest + ports: ["3000:3000"] + jobs: + build: ./jobs + command: python jobs.py +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(tt.compose), 0644) + + cfg, err := LoadCompose(dir) + if err == nil { + t.Fatalf("unsupported independent build must be refused, not flattened: config=%+v", cfg) + } + detail := strings.ToLower(err.Error()) + if !strings.Contains(detail, "jobs") || !strings.Contains(detail, "build") { + t.Fatalf("refusal must identify the service and its build, got: %v", err) + } + if !strings.Contains(detail, "./jobs") { + t.Fatalf("refusal must name the unsupported build context, got: %v", err) + } + }) + } +} From f2e8c190b711e5fef61d84e9a1d6622efcadc4e4 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:28:08 -0700 Subject: [PATCH 02/12] =?UTF-8?q?fix(config):=20compose=20importer=20field?= =?UTF-8?q?=20classification=20=E2=80=94=20translate=20healthcheck,=20reje?= =?UTF-8?q?ct=20semantic-loss=20fields=20(C05)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-strict unmarshal silently ignored healthcheck, networks, secrets, configs, profiles, deploy, env_file and security fields, importing files while dropping their semantics. Declared classification: healthcheck translates to health (web; disable forms for workers), no-op values are tolerated (networks:[default], restart always/unless-stopped, deploy replicas:1, empty everything), non-default profiles skip the service (compose semantics), metadata is ignored with reasons, and everything with semantics teploy cannot preserve is rejected naming service+field before any effect. 42 red subtests demonstrated the silent-ignore defect first; mutation check on the healthcheck mapping. --- AUDIT_OPEN.md | 44 ++ internal/config/compose.go | 428 ++++++++++++++- internal/config/compose_test.go | 920 ++++++++++++++++++++++++++++++++ 3 files changed, 1386 insertions(+), 6 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 20113da..86a5aeb 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -808,3 +808,47 @@ intended reason. Real Docker port behavior remains a later journey gate (J05); no remote deployment was performed. The preview collision and full Compose breadth stay open under the product programme (`_internal/TEPLOY_PRODUCT_EXCELLENCE_PROGRAMME_2026-09-21.md` C05/C06). + +- **Compose field contract (preserve / translate / reject)** — the + C05 defect class removed for the highest-impact fields: the importer + decoded with non-strict yaml.Unmarshal, so unknown Compose keys were + SILENTLY IGNORED — a file using healthcheck, networks, secrets, + configs, profiles, deploy.resources or env_file imported + "successfully" while dropping those semantics. The pass is bounded to + the inventoried fields; the classification table is declared as the + grammar in `TestLoadCompose_FieldClassificationInventory` + (compose_test.go) and on the LoadCompose/composeAppPort doc comments. + + | Compose field | Decision | + |---|---| + | healthcheck | TRANSLATE (web): exec-form `["CMD","curl"/"wget",...,"http://localhost:/"]` → `health.path` + `health.interval_seconds`; `disable: true` / `test: ["NONE"]` → `healthcheck.web.disable` (--no-healthcheck). Workers: only the disabling forms translate; other tests rejected (no per-process HTTP gate). Accessories: ignored — inert (teploy supervises via `--restart always` + running-state, never queries docker health). timeout/retries/start_period deliberately NOT translated: compose timeout is per-probe, teploy `health.timeout_seconds` is the TOTAL gate window (translating would break slow starters); retries/start_period subsumed by that window. CMD-SHELL/string forms, non-HTTP probes, wrong port, https, non-localhost hosts, query strings, sub-second intervals: rejected naming the service. | + | networks | REJECT except exact no-op (`[default]`, `{default: {}}`) — teploy runs every container on its own managed network | + | restart | TOLERATE `always`/`unless-stopped` (teploy runs app containers `--restart unless-stopped`, docker.go; accessories `--restart always`; the `always` delta is only after manual stop + daemon restart, which teploy's lifecycle owns); `no`/`on-failure`/other REJECTED (crash-semantics change) | + | env_file | REJECT (opaque file reference with compose-specific interpolation the importer cannot resolve; teploy `env_files` is a deliberate teploy.yml opt-in); empty tolerated | + | secrets / configs | REJECT (no model); empty tolerated | + | profiles | non-default-profile services SKIPPED entirely — `docker compose up` without `--profile` does not deploy them, so importing them would deploy something compose would not | + | extends | REJECT (inheritance not losslessly resolvable) | + | deploy | only no-op defaults tolerated (`{}`, `replicas: 1`, `mode: replicated`); resources/replicas≠1/global REJECTED | + | labels | IGNORE (container metadata; teploy manages its own teploy.* labels) | + | depends_on | TOLERATED deliberately: parsed, unused — teploy ensures every accessory is RUNNING before any app container starts (cli/deploy.go "Ensure accessories are running" step, cli/singledeploy.go), honoring the common ordering by construction; the delta (condition: service_healthy readiness gates not waited for) is documented in the table | + | container_name | REJECT (teploy owns naming, {app}-{process}-{version}) | + | hostname | REJECT (identity, no home) | + | working_dir / entrypoint | REJECT (no model home — bake into the image) | + | privileged / cap_add | false/empty tolerated; true/non-empty REJECTED (security-relevant; teploy runs unprivileged containers) | + + Accessories get the same field treatment as web (a network or + privileged on postgres is refused exactly like one on web). Evidence: + 42 new subtests; TDD red recorded (every reject/translate case + "imported successfully" against the old importer — the silent-ignore + defect demonstrated live), then green after the fix; mutation check — + making the healthcheck translation write nothing fails + `TestLoadCompose_TranslatesHealthcheck` and the inventory row with + `health.path = "", want /healthz` for the intended reason (reverted). + All refusals are import-time (LoadCompose is pure; errors propagate + through LoadApp fail-closed at deploy). + + Remaining open under C05: full Compose breadth (fields outside the + inventory are still silently ignored — KnownFields-style strictness + over the whole Compose schema), multi-image build identity, + plan/apply. Base revision `566e291`; changes left uncommitted for + review. diff --git a/internal/config/compose.go b/internal/config/compose.go index 249af66..ccdb61f 100644 --- a/internal/config/compose.go +++ b/internal/config/compose.go @@ -2,12 +2,14 @@ package config import ( "fmt" + "net/url" "os" "path/filepath" "regexp" "sort" "strconv" "strings" + "time" "github.com/useteploy/teploy/internal/ssh" "gopkg.in/yaml.v3" @@ -18,6 +20,25 @@ type composeFile struct { Services map[string]composeService `yaml:"services"` } +// composeHealthcheck mirrors the Compose healthcheck block. test is kept +// raw (list or string): the only translatable forms are the exec-list +// HTTP probe and the disabling forms, and everything else is refused +// rather than guessed at. +type composeHealthcheck struct { + Test interface{} `yaml:"test"` + Interval interface{} `yaml:"interval"` + Timeout interface{} `yaml:"timeout"` + Retries interface{} `yaml:"retries"` + StartPeriod interface{} `yaml:"start_period"` + Disable *bool `yaml:"disable"` +} + +// composeService is the subset of the Compose service schema the importer +// understands. Fields parsed here but deliberately NOT translated carry a +// comment saying where the decision lives; the full per-field +// classification table (preserve/translate/reject/tolerate/ignore, with +// the reasons) is declared in compose_test.go's +// TestLoadCompose_FieldClassificationInventory. type composeService struct { Image string `yaml:"image"` Build interface{} `yaml:"build"` // string or struct @@ -25,7 +46,37 @@ type composeService struct { Command interface{} `yaml:"command"` // string or []string Environment interface{} `yaml:"environment"` // map or list Volumes []string `yaml:"volumes"` - DependsOn interface{} `yaml:"depends_on"` // list or map + + // DependsOn is startup ORDERING under Compose. Parsed but deliberately + // not translated: teploy already ensures every accessory is running + // before any app container starts (cli/deploy.go "Ensure accessories + // are running", cli/singledeploy.go), which honors the common + // app-after-database ordering by construction. Readiness conditions + // (service_healthy) are not waited for — documented in the + // classification table. + DependsOn interface{} `yaml:"depends_on"` // list or map + + // Healthcheck translates for the web service (HTTP probe path/interval + // -> AppConfig.Health; disable/test:NONE -> + // Healthcheck["web"].Disable) and for worker disable forms; accessory + // healthchecks are inert (teploy supervises accessories itself). + Healthcheck *composeHealthcheck `yaml:"healthcheck"` + + Networks interface{} `yaml:"networks"` // tolerated only as the implicit default + Restart string `yaml:"restart"` // tolerated: always / unless-stopped (teploy's own policies) + EnvFile interface{} `yaml:"env_file"` // rejected when non-empty (opaque file reference) + Secrets []interface{} `yaml:"secrets"` // rejected when non-empty + Configs []interface{} `yaml:"configs"` // rejected when non-empty + Profiles []string `yaml:"profiles"` // non-empty: service skipped (not deployed by default compose up) + Extends interface{} `yaml:"extends"` // rejected when present + Deploy map[string]interface{} `yaml:"deploy"` // tolerated only as no-op defaults + Labels interface{} `yaml:"labels"` // deliberately ignored (container metadata) + ContainerName string `yaml:"container_name"` // rejected (teploy owns naming) + Hostname string `yaml:"hostname"` // rejected (identity, no home) + WorkingDir string `yaml:"working_dir"` // rejected (no home) + Entrypoint interface{} `yaml:"entrypoint"` // rejected (no home) + Privileged *bool `yaml:"privileged"` // true rejected (security) + CapAdd []string `yaml:"cap_add"` // non-empty rejected (security) } // knownAccessoryImages maps image prefixes to default ports. @@ -53,6 +104,16 @@ var composeFileNames = []string{ // LoadCompose reads a docker-compose file and maps it to an AppConfig. // Returns nil if no compose file is found. +// +// The importer treats Compose as a SUBSET: every supplied field it knows +// about is preserved, explicitly translated, or rejected with a named, +// actionable error — never silently dropped. The per-field classification +// (translate / reject / tolerate-the-default / ignore, with reasons and +// the model homes) is declared in compose_test.go's +// TestLoadCompose_FieldClassificationInventory; the port grammar is +// declared on composeAppPort. Widen either deliberately, never silently. +// Fields outside the classification are still ignored — full Compose +// breadth remains open under C05. func LoadCompose(dir string) (*AppConfig, error) { var data []byte for _, name := range composeFileNames { @@ -87,6 +148,18 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { cfg.App = strings.ToLower(strings.ReplaceAll(cfg.App, "_", "-")) cfg.App = strings.ReplaceAll(cfg.App, " ", "-") + // Services under non-default profiles are skipped deliberately: + // `docker compose up` without --profile does not deploy them, so + // importing them would deploy something Compose itself would not. + // Empty profiles (= no restriction) import normally. + services := make(map[string]composeService, len(compose.Services)) + for name, svc := range compose.Services { + if len(svc.Profiles) > 0 { + continue + } + services[name] = svc + } + // Find the main web service: the single non-accessory service that // publishes ports. Known accessory images are excluded even when they // publish ports — previously this loop took the FIRST service with @@ -97,7 +170,7 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { // and with more than one there is no principled automatic choice — // fail and name them rather than guess. var webCandidates []string - for name, svc := range compose.Services { + for name, svc := range services { if len(svc.Ports) > 0 && !isAccessoryImage(svc.Image) { webCandidates = append(webCandidates, name) } @@ -109,12 +182,12 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { switch { case len(webCandidates) == 1: webServiceName = webCandidates[0] - webService = compose.Services[webServiceName] + webService = services[webServiceName] case len(webCandidates) > 1: return nil, fmt.Errorf("ambiguous compose import: multiple non-accessory services publish ports (%s) — teploy cannot pick the web service automatically; remove ports from the services that are not the app, or write teploy.yml manually", strings.Join(webCandidates, ", ")) default: anyPorts := false - for _, svc := range compose.Services { + for _, svc := range services { if len(svc.Ports) > 0 { anyPorts = true break @@ -158,17 +231,46 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { cfg.Publish = extraPublish } + // Web service field contract: the same preserve/translate/reject pass + // every other service gets, plus the healthcheck translation that only + // has a home for the web process. + var violations []string + violations = append(violations, composeFieldViolations(webServiceName, webService)...) + if webService.Healthcheck != nil { + if composeHealthcheckDisabled(webService.Healthcheck) { + if cfg.Healthcheck == nil { + cfg.Healthcheck = make(map[string]ProcessHealth) + } + cfg.Healthcheck["web"] = ProcessHealth{Disable: true} + } else { + path, interval, err := translateComposeHealthcheck(webServiceName, webService.Healthcheck, webPort) + if err != nil { + violations = append(violations, err.Error()) + } else { + cfg.Health.Path = path + cfg.Health.IntervalSeconds = interval + } + } + } + // Classify remaining services. var unsupportedBuilds []string - for name, svc := range compose.Services { + for _, name := range sortedServiceNames(services) { if name == webServiceName { continue } + svc := services[name] svcBuildContext := parseBuildContext(svc.Build) // Check if it's a known accessory image. if isAccessoryImage(svc.Image) { + violations = append(violations, composeFieldViolations(name, svc)...) + // An accessory healthcheck is inert under teploy: teploy + // supervises accessories (--restart always + running-state + // checks) and never queries docker health, so it is ignored + // rather than translated or rejected (classification table). + acc := AccessoryConfig{ Image: svc.Image, Port: accessoryPort(svc.Image), @@ -192,6 +294,21 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { // Same build context as web → worker process. if svcBuildContext != "" && svcBuildContext == webBuildContext { + violations = append(violations, composeFieldViolations(name, svc)...) + if svc.Healthcheck != nil { + if composeHealthcheckDisabled(svc.Healthcheck) { + // The one worker healthcheck form with a home: suppress + // the image's HEALTHCHECK for this process (workers + // that share the web image inherit its HTTP probe, + // which fails forever for a process with no listener). + if cfg.Healthcheck == nil { + cfg.Healthcheck = make(map[string]ProcessHealth) + } + cfg.Healthcheck[name] = ProcessHealth{Disable: true} + } else { + violations = append(violations, fmt.Sprintf("compose service %q: 'healthcheck' has no home for a non-web process — teploy health-gates only the web process (teploy.yml 'health:'); only the disabling forms (disable: true / test: [\"NONE\"]) translate; remove it or write teploy.yml", name)) + } + } cfg.Processes[name] = parseCommand(svc.Command) continue } @@ -203,6 +320,7 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { // Has a standalone image that isn't a known DB → treat as accessory. if svc.Image != "" { + violations = append(violations, composeFieldViolations(name, svc)...) acc := AccessoryConfig{Image: svc.Image} env := parseEnvironment(svc.Environment) if len(env) > 0 { @@ -225,6 +343,11 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { unsupportedBuilds = append(unsupportedBuilds, fmt.Sprintf("%s (build %q)", name, svcBuildContext)) } + if len(violations) > 0 { + sort.Strings(violations) + return nil, fmt.Errorf("unsupported compose fields: %s", strings.Join(violations, "; ")) + } + if len(unsupportedBuilds) > 0 { sort.Strings(unsupportedBuilds) webSrc := fmt.Sprintf("%q builds from %q", webServiceName, webBuildContext) @@ -245,6 +368,297 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { return cfg, nil } +// composeFieldViolations returns one named, actionable refusal per supplied +// Compose field whose silent loss would change deployment semantics (the +// C05 contract: preserve, translate, or reject — never drop). Only exact +// no-op equivalents are tolerated; each check's reason names what teploy +// cannot preserve and the alternative. The full classification table is +// declared in compose_test.go. +func composeFieldViolations(name string, svc composeService) []string { + var v []string + add := func(field, why string) { + v = append(v, fmt.Sprintf("compose service %q: %s — %s; remove it or write teploy.yml", name, field, why)) + } + if !composeNetworksNoop(svc.Networks) { + add("'networks'", "teploy runs every container on its own managed network and cannot preserve Compose networks (only the implicit default, networks: [default], is a no-op)") + } + if !composeEnvFileNoop(svc.EnvFile) { + add("'env_file'", "an opaque file reference whose Compose-specific interpolation the importer cannot resolve — move the values into 'environment' or configure teploy.yml 'env_files' deliberately") + } + if len(svc.Secrets) > 0 { + add("'secrets'", "no secret-file model in the Compose import — use 'teploy secret set' or mount via 'volumes'") + } + if len(svc.Configs) > 0 { + add("'configs'", "no config-file model in the Compose import — mount the files via 'volumes'") + } + if svc.Extends != nil { + add("'extends'", "service inheritance cannot be resolved losslessly at import — inline the inherited fields") + } + if !composeDeployNoop(svc.Deploy) { + add("'deploy'", "resources/replicas/mode cannot be preserved — teploy sizes workloads via teploy.yml (replicas, memory, cpu); only the Compose defaults (replicas: 1, mode: replicated) are a no-op") + } + switch svc.Restart { + case "", "always", "unless-stopped": + // Tolerated: teploy runs app containers --restart unless-stopped + // and accessories --restart always; "always" differs from the app + // policy only after a manual stop + daemon restart, which teploy's + // own lifecycle owns (classification table). + default: + add(fmt.Sprintf("'restart: %s'", svc.Restart), "teploy supervises containers itself (app containers --restart unless-stopped, accessories --restart always); only 'always'/'unless-stopped' are equivalent, other policies change crash semantics") + } + if svc.ContainerName != "" { + add("'container_name'", "teploy owns container naming ({app}-{process}-{version}) for lifecycle management") + } + if svc.Hostname != "" { + add("'hostname'", "no teploy.yml equivalent — teploy derives container identity itself (name {app}-{process}-{version}, network alias {app}); software deriving identity from its hostname would silently change behavior") + } + if svc.WorkingDir != "" { + add("'working_dir'", "no teploy.yml equivalent — set WORKDIR in the image") + } + if !composeEntrypointNoop(svc.Entrypoint) { + add("'entrypoint'", "no teploy.yml equivalent — bake it into the image's ENTRYPOINT") + } + if svc.Privileged != nil && *svc.Privileged { + add("'privileged'", "security-relevant — teploy runs unprivileged containers and cannot preserve it") + } + if len(svc.CapAdd) > 0 { + add(fmt.Sprintf("'cap_add' (%s)", strings.Join(svc.CapAdd, ", ")), "security-relevant capabilities teploy's unprivileged containers cannot preserve") + } + return v +} + +// composeNetworksNoop reports whether a networks value is exactly the +// implicit default Compose attaches anyway: ["default"], {default: {}}, +// or absent/empty. Anything else (named networks, aliases, addresses) +// changes network semantics. +func composeNetworksNoop(v interface{}) bool { + switch n := v.(type) { + case nil: + return true + case []interface{}: + for _, e := range n { + if s, ok := e.(string); ok && s == "default" { + continue + } + return false + } + return true + case map[string]interface{}: + for k, val := range n { + if k != "default" { + return false + } + if val == nil { + continue + } + if m, ok := val.(map[string]interface{}); ok && len(m) == 0 { + continue + } + return false + } + return true + } + return false +} + +// composeEnvFileNoop reports whether an env_file value is absent or empty. +func composeEnvFileNoop(v interface{}) bool { + switch e := v.(type) { + case nil: + return true + case string: + return e == "" + case []interface{}: + for _, item := range e { + if s, ok := item.(string); ok && s == "" { + continue + } + return false + } + return true + } + return false +} + +// composeEntrypointNoop reports whether an entrypoint value is absent or +// an empty list. +func composeEntrypointNoop(v interface{}) bool { + if v == nil { + return true + } + if e, ok := v.([]interface{}); ok && len(e) == 0 { + return true + } + return false +} + +// composeDeployNoop reports whether a deploy block contains only Compose's +// own defaults ({} / replicas: 1 / mode: replicated) — values teploy's +// model already implies. Any other key or value changes deployment +// semantics and is refused. +func composeDeployNoop(m map[string]interface{}) bool { + if len(m) == 0 { + return true + } + for k, v := range m { + switch k { + case "replicas": + if n, ok := v.(int); ok && n == 1 { + continue + } + return false + case "mode": + if s, ok := v.(string); ok && s == "replicated" { + continue + } + return false + default: + return false + } + } + return true +} + +// composeHealthcheckDisabled reports whether a healthcheck block is the +// Compose disabling form: disable: true, or test: ["NONE"] (which +// suppresses the image's inherited HEALTHCHECK). Both translate to +// ProcessHealth.Disable (--no-healthcheck) — the faithful home. +func composeHealthcheckDisabled(hc *composeHealthcheck) bool { + if hc.Disable != nil && *hc.Disable { + return true + } + if elems, ok := hc.Test.([]interface{}); ok && len(elems) == 1 { + if s, ok := elems[0].(string); ok && s == "NONE" { + return true + } + } + return false +} + +// translateComposeHealthcheck maps the web service's healthcheck test to +// teploy's deploy health gate: AppConfig.Health.Path (what URL path means +// healthy) and IntervalSeconds (poll cadence). The supported grammar is +// deliberately narrow (the ParsePublishSpec precedent): exec form +// ["CMD", "curl"|"wget", ...flags..., "http://localhost:[/path]"] +// with the URL on localhost and the application port, plain http, bare +// path. timeout/retries/start_period are deliberately NOT translated — +// Compose's timeout is per-probe while teploy's health.timeout_seconds is +// the TOTAL deploy-gate window, and setting it from a per-probe value +// would break slow-starting apps; retries/start_period are subsumed by +// that total window. Everything else is refused naming the service. +func translateComposeHealthcheck(service string, hc *composeHealthcheck, appPort int) (string, int, error) { + untranslatable := func(reason string) (string, int, error) { + return "", 0, fmt.Errorf("compose service %q: 'healthcheck.test' %s — only the exec form [\"CMD\", \"curl\"|\"wget\", ..., \"http://localhost:/\"] translates to teploy's deploy health gate (teploy.yml 'health:'); remove it or write teploy.yml", service, reason) + } + elems, ok := hc.Test.([]interface{}) + if !ok { + return untranslatable("shell/string form is not imported") + } + strs := make([]string, 0, len(elems)) + for _, e := range elems { + s, ok := e.(string) + if !ok { + return untranslatable("contains a non-string element") + } + strs = append(strs, s) + } + if len(strs) < 2 || strs[0] != "CMD" { + return untranslatable("is not an exec-form command") + } + if strs[1] != "curl" && strs[1] != "wget" { + return untranslatable(fmt.Sprintf("command %q is not an HTTP probe", strs[1])) + } + var rawURL string + for _, a := range strs[2:] { + if strings.HasPrefix(a, "http://") || strings.HasPrefix(a, "https://") { + if rawURL != "" { + return untranslatable("contains more than one URL") + } + rawURL = a + } + } + if rawURL == "" { + return untranslatable("contains no URL") + } + u, err := url.Parse(rawURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return untranslatable(fmt.Sprintf("URL %q is not absolute", rawURL)) + } + if u.Scheme != "http" { + return untranslatable(fmt.Sprintf("URL %q must be plain http — teploy's gate probes plain HTTP through the published port", rawURL)) + } + switch u.Hostname() { + case "localhost", "127.0.0.1", "::1": + default: + return untranslatable(fmt.Sprintf("URL host %q must be localhost/127.0.0.1/[::1] — the container probing itself", u.Hostname())) + } + if p := u.Port(); p != "" { + if n, err := strconv.Atoi(p); err != nil || n != appPort { + return untranslatable(fmt.Sprintf("probes port %s, not the application port %d", p, appPort)) + } + } else if appPort != 80 { + return untranslatable(fmt.Sprintf("URL %q has no explicit port but the application port is %d", rawURL, appPort)) + } + if u.RawQuery != "" || u.Fragment != "" { + return untranslatable(fmt.Sprintf("URL %q must be a bare path (no query or fragment)", rawURL)) + } + path := u.Path + if path == "" { + path = "/" + } + interval := 0 + if hc.Interval != nil { + secs, err := composeDurationSeconds(hc.Interval) + if err != nil || secs < 1 { + return "", 0, fmt.Errorf("compose service %q: 'healthcheck.interval' must be a duration of whole seconds >= 1s — teploy's health.interval_seconds has no sub-second home; remove it or write teploy.yml", service) + } + interval = secs + } + return path, interval, nil +} + +// composeDurationSeconds parses a Compose duration value: bare numbers +// ("30", 30) are seconds, strings may be Go-style durations ("30s", +// "1m30s"). Fractional results are rejected by the caller (no sub-second +// home). +func composeDurationSeconds(v interface{}) (int, error) { + switch d := v.(type) { + case int: + return d, nil + case int64: + return int(d), nil + case float64: + return int(d), nil + case string: + s := strings.TrimSpace(d) + if s == "" { + return 0, fmt.Errorf("empty duration") + } + if n, err := strconv.Atoi(s); err == nil { + return n, nil + } + dur, err := time.ParseDuration(s) + if err != nil { + return 0, err + } + secs := dur.Seconds() + if secs != float64(int(secs)) { + return 0, fmt.Errorf("%s is not a whole number of seconds", s) + } + return int(secs), nil + } + return 0, fmt.Errorf("unsupported duration value %v", v) +} + +func sortedServiceNames(m map[string]composeService) []string { + names := make([]string, 0, len(m)) + for name := range m { + names = append(names, name) + } + sort.Strings(names) + return names +} + // composeAppPort resolves the web service's application container port // from its Compose ports entries, returning the port and any non-TCP // entries that must be preserved verbatim as publishes. The supported @@ -254,7 +668,9 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { // container ports are refused naming the reason — never given arbitrary // meaning. The Compose HOST-side binding is not part of the returned // value: it is host plumbing that teploy replaces with its own -// allocation and Caddy routing. +// allocation and Caddy routing. This is the ports entry of the importer's +// overall field classification — see LoadCompose and the classification +// table in compose_test.go. func composeAppPort(service string, raw []interface{}) (int, []string, error) { seen := map[int]bool{} var extra []string diff --git a/internal/config/compose_test.go b/internal/config/compose_test.go index afc8272..c466db2 100644 --- a/internal/config/compose_test.go +++ b/internal/config/compose_test.go @@ -665,3 +665,923 @@ services: }) } } + +// Compose importer field classification — the declaration of the supported +// grammar (C05: every supplied field must be preserved, explicitly +// translated, or rejected; never silently dropped). Before this slice the +// importer used non-strict yaml.Unmarshal, so every field below was +// SILENTLY IGNORED: a file using healthcheck, networks, secrets, configs, +// profiles or deploy imported "successfully" while dropping those +// semantics. +// +// Field | Before | Now +// -----------------+-----------------+------------------------------------------ +// healthcheck | silently ignored| TRANSLATE for the web service: exec-form +// | | ["CMD","curl"|"wget",...,"http://localhost: +// | | /path"] maps the path to +// | | health.path and interval to +// | | health.interval_seconds. disable: true +// | | and test: ["NONE"] map to +// | | healthcheck.web.disable (--no-healthcheck). +// | | Workers: disable/NONE translate to +// | | healthcheck..disable; other tests +// | | are rejected (teploy has no per-process +// | | HTTP gate). Accessories: ignored — inert +// | | under teploy, which supervises accessories +// | | via --restart always + running-state checks +// | | and never queries docker health. timeout/ +// | | retries/start_period are deliberately NOT +// | | translated: compose timeout is per-probe, +// | | teploy's health.timeout_seconds is the total +// | | deploy-gate window (default 30s) — setting +// | | it from a per-probe value would break +// | | slow-starting apps; retries/start_period are +// | | subsumed by that total window. +// networks | silently ignored| TOLERATE exactly the no-op equivalent: +// | | ["default"] or {default: {}} — the implicit +// | | default network compose attaches anyway. +// | | Anything else REJECTED (teploy runs every +// | | container on its own managed network). +// restart | silently ignored| TOLERATE "always"/"unless-stopped" (teploy +// | | runs app containers --restart unless-stopped, +// | | accessories --restart always; the delta for +// | | "always" is only after a manual stop + daemon +// | | restart, which teploy's lifecycle owns). +// | | Everything else ("no", "on-failure", ...) +// | | REJECTED — those change crash semantics. +// env_file | silently ignored| REJECTED: an opaque file reference with +// | | compose-specific interpolation rules the +// | | importer cannot resolve; teploy's env_files +// | | is a deliberate teploy.yml opt-in. Empty +// | | values tolerated. +// secrets | silently ignored| REJECTED (no secret-file model in the +// | | import; empty list tolerated). +// configs | silently ignored| REJECTED (no config-file model in the +// | | import; empty list tolerated). +// profiles | silently ignored| Services under non-default profiles are +// | | SKIPPED entirely, deliberately: `docker +// | | compose up` without --profile does not +// | | deploy them, so importing them would deploy +// | | something compose itself would not. +// extends | silently ignored| REJECTED (inheritance cannot be resolved +// | | losslessly). +// deploy | silently ignored| Only no-op defaults tolerated ({}, or a +// | | block containing just replicas: 1 and/or +// | | mode: replicated — compose defaults). +// | | Everything else (resources, replicas != 1, +// | | mode: global, ...) REJECTED. +// labels | silently ignored| IGNORED — container metadata with no deploy +// | | semantics; teploy manages its own teploy.* +// | | labels for lifecycle. +// depends_on | parsed, unused | TOLERATED deliberately. Compose semantics +// | | are startup ordering; teploy ensures every +// | | accessory is RUNNING before any app +// | | container starts (cli/deploy.go "Ensure +// | | accessories are running" step 9, +// | | cli/singledeploy.go — sorted, before the +// | | app containers), which honors the common +// | | app-after-db ordering by construction. The +// | | delta: condition: service_healthy / +// | | service_completed_successfully readiness +// | | gates are NOT waited for — the app must +// | | tolerate an unreachable dependency at boot +// | | (teploy's deploy health gate still gates +// | | traffic). +// container_name | silently ignored| REJECTED — teploy owns container naming +// | | ({app}-{process}-{version}) for lifecycle. +// hostname | silently ignored| REJECTED — identity with no model home; +// | | software deriving identity from hostname +// | | would silently change behavior. +// working_dir | silently ignored| REJECTED — no model home (set WORKDIR in +// | | the image). +// entrypoint | silently ignored| REJECTED — no model home (bake into the +// | | image's ENTRYPOINT). +// privileged | silently ignored| false (explicit default) tolerated; true +// | | REJECTED — security-relevant, teploy runs +// | | unprivileged containers. +// cap_add | silently ignored| Empty list tolerated; non-empty REJECTED — +// | | security-relevant capabilities. +// +// Fields outside this table (the rest of the Compose spec) are still +// silently ignored — full Compose breadth remains open under C05. +func TestLoadCompose_FieldClassificationInventory(t *testing.T) { + tests := []struct { + name string + compose string + wantErr []string // substrings the error must contain; empty = must import + check func(t *testing.T, cfg *AppConfig) + }{ + { + name: "healthcheck web translates", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/healthz"] + interval: 10s +`, + check: func(t *testing.T, cfg *AppConfig) { + if cfg.Health.Path != "/healthz" { + t.Errorf("health.path = %q, want /healthz", cfg.Health.Path) + } + if cfg.Health.IntervalSeconds != 10 { + t.Errorf("health.interval_seconds = %d, want 10", cfg.Health.IntervalSeconds) + } + }, + }, + { + name: "healthcheck web disable translates", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + disable: true +`, + check: func(t *testing.T, cfg *AppConfig) { + if !cfg.Healthcheck["web"].Disable { + t.Errorf("healthcheck.web.disable = false, want true") + } + }, + }, + { + name: "networks non-default rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + networks: [frontend] +`, + wantErr: []string{"web", "networks"}, + }, + { + name: "restart always tolerated", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + restart: always +`, + }, + { + name: "restart no rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + restart: "no" +`, + wantErr: []string{"web", "restart"}, + }, + { + name: "env_file rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + env_file: .env +`, + wantErr: []string{"web", "env_file"}, + }, + { + name: "secrets rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + secrets: [db_password] +`, + wantErr: []string{"web", "secrets"}, + }, + { + name: "configs rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + configs: [app_config] +`, + wantErr: []string{"web", "configs"}, + }, + { + name: "profiles skipped deliberately", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + migrate: + image: migrate/migrate:v4 + profiles: [tools] +`, + check: func(t *testing.T, cfg *AppConfig) { + if _, ok := cfg.Accessories["migrate"]; ok { + t.Errorf("profiled service must be skipped, got accessory %v", cfg.Accessories) + } + }, + }, + { + name: "extends rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + extends: + service: base +`, + wantErr: []string{"web", "extends"}, + }, + { + name: "deploy resources rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + deploy: + resources: + limits: + memory: 512M +`, + wantErr: []string{"web", "deploy"}, + }, + { + name: "deploy replicas rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + deploy: + replicas: 3 +`, + wantErr: []string{"web", "deploy"}, + }, + { + name: "labels ignored", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + labels: + com.example.team: platform +`, + }, + { + name: "depends_on tolerated", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + depends_on: [db] + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: pass +`, + check: func(t *testing.T, cfg *AppConfig) { + if _, ok := cfg.Accessories["db"]; !ok { + t.Fatal("expected db accessory under tolerated depends_on") + } + }, + }, + { + name: "container_name rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + container_name: my-app +`, + wantErr: []string{"web", "container_name"}, + }, + { + name: "hostname rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + hostname: app-1 +`, + wantErr: []string{"web", "hostname"}, + }, + { + name: "working_dir rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + working_dir: /srv/app +`, + wantErr: []string{"web", "working_dir"}, + }, + { + name: "entrypoint rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + entrypoint: ["/bin/sh", "-c"] +`, + wantErr: []string{"web", "entrypoint"}, + }, + { + name: "privileged rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + privileged: true +`, + wantErr: []string{"web", "privileged"}, + }, + { + name: "cap_add rejected", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + cap_add: [NET_ADMIN] +`, + wantErr: []string{"web", "cap_add"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(tt.compose), 0644) + + cfg, err := LoadCompose(dir) + if len(tt.wantErr) > 0 { + if err == nil { + t.Fatalf("expected refusal, imported: %+v", cfg) + } + for _, sub := range tt.wantErr { + if !strings.Contains(err.Error(), sub) { + t.Errorf("error must contain %q, got: %v", sub, err) + } + } + return + } + if err != nil { + t.Fatalf("expected import, got: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil config") + } + if tt.check != nil { + tt.check(t, cfg) + } + }) + } +} + +// TestLoadCompose_TranslatesHealthcheck: the web service's healthcheck has +// real homes in AppConfig — health.path/interval_seconds for an HTTP probe, +// healthcheck..disable for the disabling forms. The translation is +// narrow on purpose (repo precedent: ParsePublishSpec) — anything the +// model cannot represent faithfully is refused naming the service. +func TestLoadCompose_TranslatesHealthcheck(t *testing.T) { + tests := []struct { + name string + compose string + wantErr []string + check func(t *testing.T, cfg *AppConfig) + }{ + { + name: "exec curl with interval, timeout deliberately not translated", + compose: ` +services: + web: + image: example/web:v1 + ports: ["8080:3000"] + healthcheck: + test: ["CMD", "curl", "-fsSL", "http://localhost:3000/healthz"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 20s +`, + check: func(t *testing.T, cfg *AppConfig) { + if cfg.Health.Path != "/healthz" { + t.Errorf("health.path = %q, want /healthz", cfg.Health.Path) + } + if cfg.Health.IntervalSeconds != 10 { + t.Errorf("health.interval_seconds = %d, want 10", cfg.Health.IntervalSeconds) + } + // compose timeout is per-probe; teploy's timeout_seconds is + // the TOTAL deploy-gate window — translating 5s would cap + // the whole gate at 5s and break slow starters. + if cfg.Health.TimeoutSeconds != 0 { + t.Errorf("health.timeout_seconds = %d, want 0 (not translated from compose per-probe timeout)", cfg.Health.TimeoutSeconds) + } + }, + }, + { + name: "exec wget spider", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/health"] +`, + check: func(t *testing.T, cfg *AppConfig) { + if cfg.Health.Path != "/health" { + t.Errorf("health.path = %q, want /health", cfg.Health.Path) + } + }, + }, + { + name: "interval as bare seconds number", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/health"] + interval: 10 +`, + check: func(t *testing.T, cfg *AppConfig) { + if cfg.Health.IntervalSeconds != 10 { + t.Errorf("health.interval_seconds = %d, want 10", cfg.Health.IntervalSeconds) + } + }, + }, + { + name: "url without path maps to root", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000"] +`, + check: func(t *testing.T, cfg *AppConfig) { + if cfg.Health.Path != "/" { + t.Errorf("health.path = %q, want /", cfg.Health.Path) + } + }, + }, + { + name: "test NONE disables", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["NONE"] +`, + check: func(t *testing.T, cfg *AppConfig) { + if !cfg.Healthcheck["web"].Disable { + t.Errorf("healthcheck.web.disable = false, want true") + } + if cfg.Health.Path != "" { + t.Errorf("health.path = %q, want empty under disabled healthcheck", cfg.Health.Path) + } + }, + }, + { + name: "worker disable translates to per-process no-healthcheck", + compose: ` +services: + web: + build: . + ports: ["3000:3000"] + worker: + build: . + command: npm run worker + healthcheck: + test: ["NONE"] +`, + check: func(t *testing.T, cfg *AppConfig) { + if !cfg.Healthcheck["worker"].Disable { + t.Errorf("healthcheck.worker.disable = false, want true") + } + }, + }, + { + name: "accessory healthcheck inert", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: pass + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s +`, + check: func(t *testing.T, cfg *AppConfig) { + if cfg.Health.Path != "" || len(cfg.Healthcheck) != 0 { + t.Errorf("accessory healthcheck must be inert, got health=%+v healthcheck=%v", cfg.Health, cfg.Healthcheck) + } + }, + }, + { + name: "cmd-shell form refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:3000/health || exit 1"] +`, + wantErr: []string{"web", "healthcheck"}, + }, + { + name: "string test form refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: curl -f http://localhost:3000/health +`, + wantErr: []string{"web", "healthcheck"}, + }, + { + name: "non-http probe refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] +`, + wantErr: []string{"web", "healthcheck"}, + }, + { + name: "wrong port refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["8080:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] +`, + wantErr: []string{"web", "healthcheck", "3000"}, + }, + { + name: "https refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "https://localhost:3000/health"] +`, + wantErr: []string{"web", "healthcheck"}, + }, + { + name: "external host refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://example.com/health"] +`, + wantErr: []string{"web", "healthcheck"}, + }, + { + name: "two urls refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/a", "http://localhost:3000/b"] +`, + wantErr: []string{"web", "healthcheck"}, + }, + { + name: "query string refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/health?ready"] +`, + wantErr: []string{"web", "healthcheck"}, + }, + { + name: "sub-second interval refused", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/health"] + interval: 500ms +`, + wantErr: []string{"web", "interval"}, + }, + { + name: "worker http test refused (no per-process gate)", + compose: ` +services: + web: + build: . + ports: ["3000:3000"] + worker: + build: . + command: npm run worker + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/health"] +`, + wantErr: []string{"worker", "healthcheck"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(tt.compose), 0644) + + cfg, err := LoadCompose(dir) + if len(tt.wantErr) > 0 { + if err == nil { + t.Fatalf("expected refusal, imported: %+v", cfg) + } + for _, sub := range tt.wantErr { + if !strings.Contains(err.Error(), sub) { + t.Errorf("error must contain %q, got: %v", sub, err) + } + } + return + } + if err != nil { + t.Fatalf("expected import, got: %v", err) + } + if tt.check != nil { + tt.check(t, cfg) + } + }) + } +} + +// TestLoadCompose_RejectsSemanticFields: fields whose silent loss changes +// deployment semantics are refused BEFORE any effect, naming the service, +// the field, why teploy cannot preserve it, and the teploy.yml alternative. +// Accessory services get the same treatment — a network on postgres is +// lost exactly as silently as one on web. +func TestLoadCompose_RejectsSemanticFields(t *testing.T) { + tests := []struct { + name string + compose string + wantErr []string + }{ + { + name: "networks on accessory", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + db: + image: postgres:16 + networks: [backend] +`, + wantErr: []string{"db", "networks"}, + }, + { + name: "networks map form with alias", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + networks: + default: + aliases: [app-1] +`, + wantErr: []string{"web", "networks"}, + }, + { + name: "secrets on accessory", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + db: + image: postgres:16 + secrets: [db_cert] +`, + wantErr: []string{"db", "secrets"}, + }, + { + name: "restart on-failure", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + restart: on-failure +`, + wantErr: []string{"web", "restart", "on-failure"}, + }, + { + name: "deploy mode global", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + deploy: + mode: global +`, + wantErr: []string{"web", "deploy"}, + }, + { + name: "privileged on accessory", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + sidecar: + image: busybox:1 + privileged: true +`, + wantErr: []string{"sidecar", "privileged"}, + }, + { + name: "entrypoint string form", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + entrypoint: /docker-entrypoint.sh +`, + wantErr: []string{"web", "entrypoint"}, + }, + { + name: "env_file list form", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + env_file: + - .env.shared + - .env.local +`, + wantErr: []string{"web", "env_file"}, + }, + { + name: "extends string form", + compose: ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + extends: base +`, + wantErr: []string{"web", "extends"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(tt.compose), 0644) + + cfg, err := LoadCompose(dir) + if err == nil { + t.Fatalf("expected refusal, imported: %+v", cfg) + } + if !strings.Contains(err.Error(), "teploy.yml") { + t.Errorf("refusal must point at teploy.yml, got: %v", err) + } + for _, sub := range tt.wantErr { + if !strings.Contains(err.Error(), sub) { + t.Errorf("error must contain %q, got: %v", sub, err) + } + } + }) + } +} + +// TestLoadCompose_ToleratesNoOpEquivalents: values that are exact no-ops +// under Compose semantics import unchanged — this is deliberate +// tolerance of the DEFAULT case only, not acceptance of the field. +func TestLoadCompose_ToleratesNoOpEquivalents(t *testing.T) { + compose := ` +services: + web: + image: example/web:v1 + ports: ["3000:3000"] + networks: [default] + restart: unless-stopped + privileged: false + cap_add: [] + secrets: [] + configs: [] + env_file: [] + deploy: + replicas: 1 + mode: replicated + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: pass + networks: + default: {} + restart: always + deploy: {} +` + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(compose), 0644) + + cfg, err := LoadCompose(dir) + if err != nil { + t.Fatalf("no-op equivalents must import, got: %v", err) + } + if cfg.Port != 3000 { + t.Errorf("port = %d, want 3000", cfg.Port) + } + if _, ok := cfg.Accessories["db"]; !ok { + t.Fatal("expected db accessory") + } +} + +// TestLoadCompose_IgnoresMetadataFields: labels and depends_on have no +// deployment semantics teploy loses — labels are container metadata, and +// depends_on's startup ordering is honored by construction (accessories +// are ensured running before any app container starts; see +// cli/deploy.go "Ensure accessories are running"). The readiness-condition +// delta is documented in the classification table above. +func TestLoadCompose_IgnoresMetadataFields(t *testing.T) { + compose := ` +services: + web: + build: . + ports: ["3000:3000"] + labels: + - "com.example.owner=platform" + - "com.example.service=web" + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: pass + redis: + image: redis:7 +` + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(compose), 0644) + + cfg, err := LoadCompose(dir) + if err != nil { + t.Fatalf("metadata fields must import, got: %v", err) + } + if _, ok := cfg.Accessories["db"]; !ok { + t.Fatal("expected db accessory") + } + if _, ok := cfg.Accessories["redis"]; !ok { + t.Fatal("expected redis accessory") + } + if cfg.Processes["web"] != "" || len(cfg.Processes) != 0 { + t.Errorf("processes = %v, want collapsed single empty-command web (nil map)", cfg.Processes) + } +} From 22ae8017f863ad6027c03729c9c364396916f352 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:49:22 -0700 Subject: [PATCH 03/12] =?UTF-8?q?feat(deploy):=20C01=20recovery=20state=20?= =?UTF-8?q?table=20=E2=80=94=20exhaustive=20crash=20dispositions=20+=20fau?= =?UTF-8?q?lt=20harness=20(design=20spike)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight-state deploy lifecycle with a pure Decide() function over (state, observed-target-evidence) returning retry/inspect/compensate/ manual, tested over the full 472,392-pair product space under seven safety invariants. ADR maps the lattice onto the existing fenced-lock/ releasemeta machinery and records ten disagreements (C01-1..C01-10) between current code dispositions and the table — the implementation roadmap for C01's code slices. Integration-tagged fault harness for a real SSH+Docker host (delayed-effect-after-owner-death, crash-after- effect-before-receipt, abandoned owner); skips cleanly without TEPLOY_FAULT_HOST. No existing deploy paths modified. --- AUDIT_OPEN.md | 98 ++++ docs/C01_RECOVERY_STATE_TABLE.md | 312 ++++++++++++ .../recovery/harness_integration_test.go | 416 ++++++++++++++++ internal/deploy/recovery/recovery.go | 456 ++++++++++++++++++ internal/deploy/recovery/recovery_test.go | 423 ++++++++++++++++ 5 files changed, 1705 insertions(+) create mode 100644 docs/C01_RECOVERY_STATE_TABLE.md create mode 100644 internal/deploy/recovery/harness_integration_test.go create mode 100644 internal/deploy/recovery/recovery.go create mode 100644 internal/deploy/recovery/recovery_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 86a5aeb..c2262f1 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -852,3 +852,101 @@ and full Compose breadth stay open under the product programme over the whole Compose schema), multi-image build identity, plan/apply. Base revision `566e291`; changes left uncommitted for review. + +## Programme slice (2026-09-21) — C01 crash-recovery state table + +Workstream C01 (P0): the implementation handoff's crash-recovery design +obligations landed as a bounded DESIGN+CODE slice. Base revision `f2e8c19`; +changes left uncommitted for review. No deploy code path was modified. + +**Landed:** + +- **The transition table as tested code** — `internal/deploy/recovery`: + the eight lifecycle states (admitted → prepared → candidates-running → + readiness-passed → traffic-switched → authoritative-state-committed → + predecessor-retired → terminal-receipt-persisted), the transition + lattice with per-transition durable-evidence citations (exact container + names/labels, Caddy marker-block + reload + delivery receipts, fenced + state.json rename, releasemeta record refs, attempt dirs), and + `Decide(from, observation)` — a pure, total crash-window disposition + function (RETRY / INSPECT / COMPENSATE / MANUAL) over tri-state + evidence. Tested exhaustively: every state × every 3^10 evidence + combination with cross-cutting safety invariants, plus canonical + per-window dispositions and the handoff's named conflicts (candidate + running with route never switched → INSPECT; predecessor already + retired under uncommitted traffic → MANUAL; unknown container names → + MANUAL). The exhaustive invariants caught two real rule-ordering + defects during development (record/target mismatch upgrading the + proven-dark window from MANUAL to INSPECT; unreadable side evidence + overriding proven-dark) — the ordering is now R0-R7 with the + proven-dark MANUAL ahead of both. +- **ADR** — `docs/C01_RECOVERY_STATE_TABLE.md`: mermaid lattice, + disposition rules, the mapping onto the existing fenced-lock / + releasemeta / attempt machinery (what already agrees), the multi-host + rule (sequence of recorded outcomes + per-generation compensation; no + global atomic commit), the lock-ordering rule (per-host app fences + + short shared-proxy commit lock; never hold one host's lock waiting on + another — current code complies), and the findings below. +- **Fault-prototype harness** — + `internal/deploy/recovery/harness_integration_test.go` + (`//go:build integration`, the repo's first integration-tagged test): + drives a real SSH+Docker host (TEPLOY_FAULT_HOST/USER/KEY; skips with + a clear message when unset) through the handoff's decisive scenarios — + (a) a nohup'd docker effect landing after owner death and a genuine + stale-break acquisition by a new owner (asserts the reconciliation + decision differs from the quiescence assumption), (b) candidate + running with no state/record (asserts INSPECT, never invented + success), (c) a stale holder's guarded effect AND fenced state commit + refused by the EXISTING fence machinery (ErrFenceLost, nothing on + disk). Prints a scenario × observed × decision × correctness table. + NOT executed in this slice (no fixture host available): it compiles + (`go vet -tags integration` clean), its decision logic is the + exhaustively-tested unit code, and it skips cleanly. + +**Disagreements found between current code dispositions and the table** +(full detail with file:line in the ADR; these feed C01's implementation +slices): + +- C01-1 lock acquisition is treated as quiescence — nothing reconciles + the dead holder's in-flight effects after a stale break + (state.go:479-516; deploy.go:414-453 handles only the same-version + rename case). +- C01-2 pre-commit effects are check-then-act (`lk.Check` separate from + the effect: deploy.go:570, 661, 712) — only WriteFenced composes + guard+effect; a broken holder's candidate/route effects can land inside + the new owner's window (A05/T01's consequence, now stated as a + disposition). +- C01-3 the shared Caddy lock is ownerless/unfenced (caddy.go:684-697) — + conflicting-route evidence (→ INSPECT) has no producer or consumer + today. +- C01-4 no durable readiness receipt exists — ReadinessPassed is + unobservable post-crash and collapses into CandidatesRunning's INSPECT + (health.go; deploy.go:646-658). +- C01-5 the terminal receipt logs Success:true even when predecessor + retirement partially failed (deploy.go:976-989 warnings + + deploy.go:934 success log; LogEntry has no degraded field) — a + fleet-rollback decision keyed on the log would skip a host still + running the superseded generation. +- C01-6 record-write failure degrades silently; the table's promised + RETRY-convergence has no reconciler (recordRelease warns, + deploy.go:1230-1232; backfill fires only from rollback/recreate). +- C01-7 compensation reconstructs the predecessor route from config + + live inspect instead of the recorded receipt (deploy.go:1097-1137; + A12/T05 — the table requires undo-to-known-predecessor). +- C01-8 same-version running `_replaced` defers to the operator (MANUAL) + where the table says INSPECT→compensable (deploy.go:438-451; needs + F04/A09 generation identities — deliberate A08 containment, recorded + as disagreement not defect). +- C01-9 candidate names are version-keyed, not attempt-keyed + (docker.go:82-117) — two attempts of one hash are not attributable by + evidence; F08 attempt ids are the existing keying surface. +- C01-10 the predecessor snapshot is in-memory only (deploy.go:464-473) + — retirement re-derives correctly but loses the removed-worker + capture on crash; the journal slice should persist it. + +**Open:** harness execution against a real fixture host (next slice); +the ten findings above as C01 implementation work. Gates: `go vet ./...` +clean; `go vet -tags integration ./internal/deploy/recovery` clean; +`go test ./... -race` all packages ok (integration-tagged code excluded +by default); `go test -tags integration …TestFaultHarness` skips cleanly +with env unset; gofmt clean. diff --git a/docs/C01_RECOVERY_STATE_TABLE.md b/docs/C01_RECOVERY_STATE_TABLE.md new file mode 100644 index 0000000..6652858 --- /dev/null +++ b/docs/C01_RECOVERY_STATE_TABLE.md @@ -0,0 +1,312 @@ +# C01 — Crash-recovery state table (ADR) + +Programme workstream C01 (P0), landing the implementation handoff's +"Crash-recovery design obligations" as **tested code**: the deploy +lifecycle state table, the crash-window disposition lattice, and the +fault-prototype harness. The decision function is +`internal/deploy/recovery.Decide(from, observation)`; its exhaustive test +enumerates every state × every evidence combination (8 × 3^10). A table in +a doc alone was explicitly insufficient — this doc records the design; the +package is the contract. + +## The lattice + +```mermaid +stateDiagram-v2 + [*] --> Admitted + Admitted --> Prepared : artifacts / predecessor snapshot + Prepared --> CandidatesRunning : docker run (exact container IDs) + CandidatesRunning --> ReadinessPassed : health gate + ReadinessPassed --> TrafficSwitched : Caddy block + reload receipt + TrafficSwitched --> AuthoritativeStateCommitted : fenced state.json rename + AuthoritativeStateCommitted --> PredecessorRetired : stop snapshot + PredecessorRetired --> TerminalReceiptPersisted : meta record + log entry + TerminalReceiptPersisted --> [*] +``` + +This is `DeployFenced`'s commit order (`internal/deploy/deploy.go:235-939`), +generalized. Ingress variants: `ingress: host` publishes the fixed port on +the candidate itself (the TrafficSwitched receipt is the port binding, not +a Caddy block); `ingress: external` has no edge step at all (the transition +is a no-op and the receipt is vacuous). + +## Per-transition evidence and crash-window dispositions + +Each transition carries the durable evidence that proves it landed (names +aligned with what the code persists today) and the disposition for an owner +that dies inside the transition's window. Also encoded as data in +`recovery.Lattice()`. + +| # | Transition | Durable evidence (file:line) | Crash in window | Why | +|---|---|---|---|---| +| 1 | admitted → prepared | attempt dir `/deployments//meta/att/./` (`internal/releasemeta/attempt.go:107-116`); owner token in `.lock/info` (`internal/state/lock.go:86-92`, `internal/state/state.go:522-533`) | **RETRY** | attempt paths are random-id write-once; a fresh attempt collides with nothing | +| 2 | prepared → candidates-running | container IDs from docker run (`internal/deploy/deploy.go:577-607`); names `{app}-{process}-{version}[-{index}]` + `teploy.*` labels (`internal/docker/docker.go:82-117,160-165`) | **INSPECT** | a running candidate with no receipt is never success; corpses are reconciled by the next attempt (`deploy.go:1282-1292`) | +| 3 | candidates-running → readiness-passed | **none — no durable receipt exists** (`internal/deploy/health.go` probes are ephemeral) | **INSPECT** | unobservable post-crash; the owner must re-probe (finding C01-4) | +| 4 | readiness-passed → traffic-switched | managed marker block `# TEPLOY BEGIN `…`END` naming candidate upstreams (`internal/caddy/caddy.go:20-21,584-625`); reload receipt (`caddy.go:29-32`); delivery verification md5 host-vs-container (`caddy.go:523-550`) | **COMPENSATE** | traffic on an uncommitted generation; undo via the recorded/serving predecessor (`abortStateCommit`, `deploy.go:1036-1095`). MANUAL when the predecessor is gone | +| 5 | traffic-switched → authoritative-state-committed | fenced rename of `state.json` naming the release (`internal/state/lock.go:353-381`); `Generation`/`OperationID` (`internal/state/state.go:68-95`) | **COMPENSATE** | the commit is the single fenced atomic effect; before it, traffic is uncommitted | +| 6 | authoritative-state-committed → predecessor-retired | predecessor snapshot stopped (`internal/deploy/deploy.go:839-861,963-989`); absence in the label inventory (`internal/docker/docker.go:568-571`) | **RETRY** | retirement re-derives from the inventory; failures reported, never silent | +| 7 | predecessor-retired → terminal-receipt-persisted | record `/deployments//meta/.json` 0600 atomic (`internal/releasemeta/releasemeta.go:216-248`); log entry `/deployments/teploy.log` (`internal/state/state.go:662-679`) | **RETRY** | convergent: same-version record rewrite and live-container backfill (`releasemeta.go:336-473`) both heal a missing record | + +Which transitions can be **retried**: 1, 6, 7 (idempotent or convergent). +**Inspected**: 2, 3 (effects landed without receipts). **Compensated**: 4, +5 (undo via predecessor). **Explicit recovery decision (MANUAL)**: any +observation class the automation must not attribute — unattributable +containers, unreadable authority, traffic on an uncommitted generation +with the predecessor gone. + +## Disposition rules + +`Decide` is a pure, total function over (from-state, observed evidence). +Evidence classes are tri-state (`absent`/`present`/`unknown`) and are +PROVEN states of the target — exact container names and receipts, never +guesses. Rules in evaluation order (`recovery.go`): + +- **R0** self-contradictory bookkeeping → MANUAL +- **R1** unattributable workloads (running `teploy.app`-labeled containers + under names that are neither candidate nor predecessor) → MANUAL +- **R2** authority (state.json) unreadable → MANUAL +- **R3** traffic PROVEN on a generation the authority does not name, with + the predecessor PROVEN gone → MANUAL, regardless of what the records + claim +- **R4** any other unreadable class → INSPECT (re-observe; never + RETRY/COMPENSATE on unreadable evidence) +- **R5** edge names both generations → INSPECT (reconcile against records) +- **R6** record/target disagreement on whether the commit happened → + INSPECT (the crash was elsewhere than recorded, or an operator moved + authority via rollback — never blindly redeploy) +- **R7** authority dispatch — committed: finish the idempotent tail + (RETRY) unless the edge contradicts (INSPECT); uncommitted: traffic on + candidates → COMPENSATE, running candidate without receipt → INSPECT, + displaced predecessor with no candidate → COMPENSATE (the app is dark), + nothing landed → RETRY; no state at all: running candidate → INSPECT, + else RETRY + +The exhaustive test (`recovery_test.go`, `TestExhaustiveProductSpace`) +asserts the cross-cutting invariants over the full 8 × 59049 product space: +unattributable/unreadable-authority never auto-decided; COMPENSATE only +with a restorable predecessor under predecessor authority; RETRY only on +fully readable, non-conflicting, attributable evidence; never invented +success (an uncommitted running candidate without edge commitment is +INSPECT at minimum). + +## Mapping onto the existing machinery — what already agrees + +- **Fenced locks (F16, `internal/state/lock.go`)** provide Admitted and + the fenced state commit: `AcquireLockFenced` mints the owner token, + renewal keeps a slow owner from being falsely broken, `WriteFenced` is + the one guard+effect-composed atomic, and `ReleaseLockFenced` (A04/T02) + never deletes a successor's lock. Transition 5's disposition rests on + exactly this. +- **Unfenced compensation (deliberate, register A07)** matches the table: + `restoreDisplacedAndStarted` and `abortStateCommit` run on detached + bounded contexts (A11) after fence loss — refusing to clean up one's own + partial effects is how a fencing design strands an app. +- **`abortStateCommit` (`deploy.go:1036-1095`)** is the table's + transition-4/5 COMPENSATE, including the recreate-strategy branch + (restart displaced fixed-port workload) and A10's route-restore ordering. +- **Attempt artifacts (F08, `internal/releasemeta/attempt.go`)** make + transition 1 retryable: write-once, random-id, never rewritten. +- **Record convergence (F14, `internal/releasemeta/releasemeta.go`)** makes + transition 7 retryable: same-version rewrite is the documented + immutability exception and `Backfill` (`releasemeta.go:336-473`) + converges pre-F14 installs. +- **Caddy edit transaction (F45/F48/F49, `internal/caddy/caddy.go:437-511`)**: + adapt gate pre-write, rollback+reload-restore on failure, delivery + verification — the receipts transition 4 cites. + +## Disagreements between current code and the table (findings) + +These are the deltas the C01 implementation slices must close. Each is a +statement of where today's code's effective disposition differs from the +table's, with the register item it belongs to. + +1. **C01-1 — Lock acquisition is treated as quiescence.** + `acquireAutoLock` (`internal/state/state.go:479-516`) breaks a stale + lock and proceeds directly into a deploy; nothing reconciles in-flight + effects from the dead holder. A late `docker run` issued by the dead + owner lands under version-keyed names; a same-version collision + surfaces later as a generic docker error, not a reconciliation. The + table requires: a replacement owner runs `Decide` over observed + evidence after acquisition. Register: A05/T01 adjacent but distinct — + fencing refuses stale *check-then-act* holders; this is the new owner's + side (nobody observes the leftover world). + +2. **C01-2 — Pre-commit effects are check-then-act, not guarded.** + `lk.Check` runs as a separate command from the effect it guards: + candidate starts (`internal/deploy/deploy.go:570-577`), worker starts + (`deploy.go:661-663`), route switch (`deploy.go:712-731`). Only the + state commit composes guard+effect in one shell (`WriteFenced`, + `internal/state/lock.go:377`). Between check and effect a takeover can + occur, so a broken holder's candidate/route effects can land inside the + new owner's window — the table treats "effect lands after owner death" + as INSPECT-at-best evidence, which nothing today generates. Register: + A05/T01 standing; the table now states the disposition consequence. + +3. **C01-3 — The shared Caddy lock is ownerless and unfenced.** + `internal/caddy/caddy.go:684-697` breaks any caddy lock older than 120s + and carries no owner identity; a slow or queued edit from an orphaned + owner can interleave with the new owner's `mutate`. The table's + "conflicting route evidence → INSPECT" has no producer/consumer today + (nobody reconciles a Caddyfile that names containers no inventory can + attribute). Register: T03 documents the design as deliberate; the + finding is the missing reconciliation, not the lock's shape. + +4. **C01-4 — No durable readiness receipt.** The health gate + (`internal/deploy/health.go`, `deploy.go:646-658`) persists nothing, so + ReadinessPassed is unobservable post-crash and its recovery disposition + collapses into CandidatesRunning's INSPECT. A receipt (attempt-scoped + marker recording the probed port/time/result) is a design obligation + for the helper/journal slice. + +5. **C01-5 — The terminal receipt records success on incomplete + retirement.** Predecessor stop/remove failures are warnings + (`internal/deploy/deploy.go:976-989`) but `logDeploy(ctx, cfg, true, …)` + (`deploy.go:934`) still appends `Success: true`, and + `state.LogEntry` (`internal/state/state.go:98-114`) has no + degraded/partial field. The table (and the multi-host rule below) + requires recorded outcomes to be the real outcomes — a fleet rollback + decision keyed on that log would skip a host that is still running the + superseded generation. + +6. **C01-6 — Record-write failure degrades silently.** `recordRelease` + warns (`internal/deploy/deploy.go:1230-1232`) and nothing schedules + convergence; backfill fires only when rollback/recreate needs a record + (`internal/deploy/rollback.go:582-587`). The table says transition 7 is + RETRY-convergent — correct — but no reconciler exists: `status`/`drift` + do not heal a missing record, so the convergence the table promises is + latent until the next deploy. + +7. **C01-7 — Compensation reconstructs the predecessor instead of using a + receipt.** `restorePreviousRoute` (`internal/deploy/deploy.go:1097-1137`) + rebuilds the previous Caddy block from current config + live inspect; + rollback's route restoration follows the same shape. The table's + COMPENSATE means "undo via the KNOWN predecessor state" — the recorded + block/spec (F14 record, `ParseSites`/`ExtractPolicy` + `internal/caddy/routes.go:89,429`) — not an inference that can + compensate to the wrong block when config drifted. Register: A12/T05 + standing; the table sharpen the disposition language. + +8. **C01-8 — Same-version `_replaced` handling is MANUAL where the table + says INSPECT→compensable.** The running-`_replaced` refusal + (`internal/deploy/deploy.go:438-451`) defers to the operator + ("teploy rollback") — deliberate A08 containment. The table classifies + the world (running `_replaced` = the renamed serving predecessor) as + INSPECT with an adopt-as-predecessor continuation; automating it + requires generation-scoped identities (register F04/A09). The current + code's MANUAL is the safe subset — recorded as a disagreement, not a + defect. + +9. **C01-9 — Candidate identities are version-keyed, not + attempt/generation-keyed.** `{app}-{process}-{version}[-{index}]` + (`internal/docker/docker.go:82-117`) means two attempts of the same + release hash share candidate names; evidence attribution between them + relies on the `_replaced` convention alone. The table's "exact + container IDs" evidence requirement points at attempt-scoped names — + F08's attempt ids are the existing keying surface (register F04/A09). + +10. **C01-10 — The predecessor snapshot is in-memory only.** + `deploy.go:464-473` snapshots predecessors before candidates start, + but a crash loses it; retirement re-derives via `selectPredecessors` + (TCL-02-correct) at the cost of the removed-worker capture property. + The journal slice should persist the snapshot with the attempt + artifacts. + +## Multi-host rule + +A multi-server rollout is a **sequence of recorded outcomes and +compensation, not a fictitious globally atomic commit.** Concretely, and +as the code already shapes it (`internal/cli/deploy.go:807-841` canary + +main waves, `internal/multideploy/multideploy.go` parallel slots): + +- Each host runs the FULL lifecycle (all eight states) under its own + per-host fenced lock; the per-host terminal receipts (state.json + generation, meta record, log entry) are the rollout's record of + outcomes. +- Failure handling is per-generation compensation: failed canary waves + roll back (`rollbackFailedWave`, `internal/cli/deploy.go:953`), and a + post-wave failure rolls back every succeeded host + (`internal/cli/deploy.go:917`) — except within the failure budget, + where stragglers are reported, never silently yo-yo'd. +- The load-balancer activation after a wave is a required phase (T57 + `internal/cli/deploy.go`): nonzero exit on LB failure, never a silent + partial success. +- There is no cross-host commit coordinator and none is planned: the + table's dispositions apply per host, and the fleet-level "state" is the + union of recorded outcomes plus the compensation decisions made from + them. This also means C01-5 (success logged on incomplete retirement) + is a fleet-correctness bug, not cosmetic. + +## Lock-ordering rule + +Two lock layers, never nested across hosts: + +1. **App-level fenced locks** (`/deployments//.lock`, owner token + + renewal): serialize an app's lifecycle on ONE target. Held for the + whole lifecycle, but only ever against one host. +2. **The shared-proxy commit lock** (`/deployments/caddy/.lock`, + `internal/caddy/caddy.go:684-707`): short-lived, ownerless by design + (T03), held only for the brief Caddyfile edit+reload+verify inside one + host's traffic-switch step. + +**Never hold one host's app lock while waiting on another host's.** The +current code complies: locks are acquired inside each host's +`Deploy`/`DeployFenced` (per-host executors), waves run to completion +before the LB step, and the LB activation holds no app locks. Any future +cross-host orchestration must preserve this — a deploy holding host A's +lock while blocked on host B's turn converts B's outage into A's, and a +stale-break on A mid-wait is exactly the abandoned-owner scenario the +fence exists to refuse. + +## The fault harness + +`internal/deploy/recovery/harness_integration_test.go` +(`//go:build integration`, excluded from default `go test ./...`). +Skips cleanly when the env is unset. It is a test rather than `cmd/` +because it is fixture-gated verification, not a shipped binary (CLAUDE.md's +integration-test convention; there are still no other integration-tagged +tests — this is the first, and the pattern is now established). + +Invocation once a fixture host exists: + +```sh +TEPLOY_FAULT_HOST=10.0.0.5 \ +TEPLOY_FAULT_USER=root \ +TEPLOY_FAULT_KEY=~/.ssh/id_ed25519 \ +go test -tags integration -run TestFaultHarness -v ./internal/deploy/recovery +``` + +Host prerequisites: Docker reachable by the SSH user without sudo, the +fixture image pullable (default `alpine:3`, override `TEPLOY_FAULT_IMAGE`), +writable `/deployments`. Caddy optional. **Disposable fixture only** — the +harness creates and removes `/deployments/-{a,b,c}` (prefix +override `TEPLOY_FAULT_APP`, default `faultprobe`). + +Scenarios (each prints a scenario × observed × decision × correctness row): + +- **(a) Delayed effect after owner death** — owner A takes the real fenced + lock, launches a nohup'd `sleep 4; docker run …` (a candidate-shaped + container of release `deadgen`), and dies (session closed). The lock is + aged past `staleLockTTL` (owner token preserved) and owner B acquires + through the genuine stale-break path. The late container lands after B's + acquisition; B's observation shows an unattributable running container + and `Decide` returns MANUAL — different from the quiescence assumption's + RETRY, proving reconciliation detects the late effect rather than + assuming lock acquisition proves quiescence. +- **(b) Side effect without receipt** — a candidate container running + under exact candidate naming/labels, no state.json, no record: `Decide` + returns INSPECT from `CandidatesRunning`, never invented success. +- **(c) Stale holder's late write** — successor breaks the aged lock via + the real machinery; the stale holder's `Guarded` effect AND + `WriteFenced` state commit are refused with `ErrFenceLost`, and neither + the marker file nor state.json exists. Uses only the existing fence + machinery. + +## Status + +Landed in this slice: the table (tested, exhaustive), this ADR, the +harness (compiles, unit-tested decision logic, skips without a fixture). +Open: **execution against a real fixture** (next slice, once the fixture +host exists), and the ten disagreement findings above feed C01's +implementation slices (recovery owner on acquisition, guarded pre-commit +effects, readiness receipt, honest terminal receipts, receipt-driven +compensation, attempt-scoped identities). diff --git a/internal/deploy/recovery/harness_integration_test.go b/internal/deploy/recovery/harness_integration_test.go new file mode 100644 index 0000000..6a7cd88 --- /dev/null +++ b/internal/deploy/recovery/harness_integration_test.go @@ -0,0 +1,416 @@ +//go:build integration + +// Fault-prototype harness (programme workstream C01). NOT run by default: +// this file carries the repo's integration-test build tag (CLAUDE.md +// convention), excluded from `go test ./...`, and drives a REAL SSH+Docker +// host through the implementation handoff's three decisive crash scenarios: +// +// (a) delayed command completing after owner death — a docker effect +// issued by an owner whose process dies lands AFTER a new owner has +// broken the stale lock and acquired it; the new owner's +// reconciliation (Decide over observed evidence) must detect the late +// effect instead of assuming lock acquisition proves quiescence. +// (b) crash after side effect but before receipt — candidate containers +// running, no state.json, no release record: the decision function +// must return the reconciliation disposition (INSPECT), never +// invented success. +// (c) abandoned owner (fence break) — a stale holder's late write must +// be refused by the EXISTING fence machinery (lock.Guarded / +// state.WriteFenced → ErrFenceLost), using no harness-local locking. +// +// Invocation (fixture host required — running it is the NEXT slice; this +// slice proves it compiles and skips cleanly): +// +// TEPLOY_FAULT_HOST=10.0.0.5 \ +// TEPLOY_FAULT_USER=root \ +// TEPLOY_FAULT_KEY=~/.ssh/id_ed25519 \ +// go test -tags integration -run TestFaultHarness -v ./internal/deploy/recovery +// +// Host prerequisites: Docker reachable by the SSH user (no sudo), the +// fixture image pullable (default alpine:3, override TEPLOY_FAULT_IMAGE), +// and a writable /deployments (the harness creates and removes +// /deployments/-{a,b,c} — run against a DISPOSABLE fixture only). +// Caddy is optional: the route evidence classes read Absent when no +// /deployments/caddy/Caddyfile exists. +// +// It is a test (not cmd/) because it is fixture-gated verification, not a +// shipped binary — matching how CLAUDE.md scopes integration tests. +package recovery + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +const faultSkipMsg = "fault harness needs a real SSH+Docker fixture: set TEPLOY_FAULT_HOST, TEPLOY_FAULT_USER, TEPLOY_FAULT_KEY (see internal/deploy/recovery/harness_integration_test.go header and docs/C01_RECOVERY_STATE_TABLE.md)" + +type faultEnv struct { + host string + user string + key string + app string // app-name prefix for the scenario fixtures + image string +} + +func faultEnvFrom(t *testing.T) faultEnv { + t.Helper() + host := os.Getenv("TEPLOY_FAULT_HOST") + user := os.Getenv("TEPLOY_FAULT_USER") + key := os.Getenv("TEPLOY_FAULT_KEY") + if host == "" || user == "" || key == "" { + t.Skip(faultSkipMsg) + } + e := faultEnv{ + host: host, + user: user, + key: key, + app: os.Getenv("TEPLOY_FAULT_APP"), + image: os.Getenv("TEPLOY_FAULT_IMAGE"), + } + if e.app == "" { + e.app = "faultprobe" + } + if e.image == "" { + e.image = "alpine:3" + } + return e +} + +func faultConnect(t *testing.T, env faultEnv, label string) *ssh.RemoteExecutor { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + exec, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: env.host, User: env.user, KeyPath: env.key}) + if err != nil { + t.Fatalf("connecting %s session to %s@%s: %v", label, env.user, env.host, err) + } + t.Cleanup(func() { exec.Close() }) + return exec +} + +// ageLockStale simulates the stale window elapsing: the lock info keeps +// its ORIGINAL owner token (the dead holder's fencing identity) but its +// timestamp moves past staleLockTTL, so the next AcquireLockFenced walks +// the genuine stale-break path — ReleaseLock, mkdir, fresh owner token. +func ageLockStale(t *testing.T, exec ssh.Executor, app, owner string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + old := time.Now().UTC().Add(-31 * time.Minute).Format(time.RFC3339) + info := fmt.Sprintf("{\"type\":\"auto\",\"owner\":%q,\"ts\":%q}\n", owner, old) + path := fmt.Sprintf("/deployments/%s/.lock/info", app) + if err := exec.Upload(ctx, strings.NewReader(info), path, "0644"); err != nil { + t.Fatalf("aging %s's lock: %v", app, err) + } +} + +func fileExists(t *testing.T, exec ssh.Executor, path string) bool { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + out, err := exec.Run(ctx, fmt.Sprintf("test -e %s && echo yes || echo no", ssh.ShellQuote(path))) + if err != nil { + t.Fatalf("probing %s: %v", path, err) + } + return strings.TrimSpace(out) == "yes" +} + +func waitRunning(t *testing.T, exec ssh.Executor, name string, timeout time.Duration) bool { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + out, err := exec.Run(ctx, "docker inspect -f '{{.State.Status}}' "+ssh.ShellQuote(name)+" 2>/dev/null || true") + cancel() + if err == nil && strings.TrimSpace(out) == "running" { + return true + } + time.Sleep(500 * time.Millisecond) + } + return false +} + +// observe collects a recovery.Observation from the live host with exact +// names and receipts (no guessed identities): the docker label inventory, +// state.json, the managed Caddy block, and the per-release record. This is +// the evidence-collection half the C01 helper will productionize; it lives +// in the harness so the decision package stays pure. +func observe(ctx context.Context, exec ssh.Executor, app, attempted, predecessor string) Observation { + var o Observation + + dk := docker.NewClient(exec) + containers, err := dk.ListContainers(ctx, app) + if err != nil { + o.Candidates, o.CandidateCorpses, o.ForeignCandidates = Unknown, Unknown, Unknown + } else { + candPrefix := fmt.Sprintf("%s-web-%s", app, attempted) + predName := "" + if predecessor != "" { + predName = fmt.Sprintf("%s-web-%s", app, predecessor) + } + for _, c := range containers { + isCandidate := strings.HasPrefix(c.Name, candPrefix) + isPredecessor := predName != "" && + (strings.HasPrefix(c.Name, predName) || strings.HasPrefix(c.Name, predName+"_replaced")) + switch { + case isCandidate: + if c.State == "running" { + o.Candidates = Present + } else { + o.CandidateCorpses = Present + } + case isPredecessor: + if strings.HasSuffix(c.Name, "_replaced") || c.State == "running" { + o.PredecessorServing = Present + } else { + o.PredecessorStopped = Present + } + default: + // Neither the attempted release's candidate naming nor the + // known predecessor's: an unattributable workload — only + // RUNNING ones consume traffic/jobs. + if c.State == "running" { + o.ForeignCandidates = Present + } + } + } + } + + st, err := state.Read(ctx, exec, app) + switch { + case err != nil: + o.StateToCandidate, o.StateToPredecessor = Unknown, Unknown + case st == nil: + o.StateToCandidate, o.StateToPredecessor = Absent, Absent + case st.CurrentHash == attempted: + o.StateToCandidate = Present + default: + o.StateToPredecessor = Present + } + + caddyfile, present, err := state.ReadRemoteFile(ctx, exec, "/deployments/caddy/Caddyfile") + switch { + case err != nil: + o.RouteToCandidate, o.RouteToPredecessor = Unknown, Unknown + case !present: + o.RouteToCandidate, o.RouteToPredecessor = Absent, Absent + case strings.Contains(string(caddyfile), fmt.Sprintf("%s-web-%s", app, attempted)): + o.RouteToCandidate = Present + default: + o.RouteToPredecessor = Present + } + + rec, err := releasemeta.Read(ctx, exec, app, attempted) + switch { + case err != nil: + o.ReleaseRecord = Unknown + case rec != nil: + o.ReleaseRecord = Present + default: + o.ReleaseRecord = Absent + } + return o +} + +// TestFaultHarness drives the handoff's decisive scenarios against a real +// host and prints the scenario × observed × decision × correctness result +// table. Each scenario is independent (own app dir); failures are reported +// per row, not silently aggregated. +func TestFaultHarness(t *testing.T) { + env := faultEnvFrom(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Fixture sanity: docker must answer and the image must be pullable. + probe := faultConnect(t, env, "probe") + if out, err := probe.Run(ctx, "docker version --format '{{.Server.Version}}'"); err != nil { + t.Skipf("fixture host has no reachable docker daemon (%v) — the harness requires SSH+Docker", err) + } else { + t.Logf("fixture docker server %s", strings.TrimSpace(out)) + } + if _, err := probe.Run(ctx, "docker pull "+ssh.ShellQuote(env.image)); err != nil { + t.Skipf("cannot pull fixture image %s (%v) — pre-pull it or set TEPLOY_FAULT_IMAGE", env.image, err) + } + + type row struct { + scenario, observed, decision string + pass bool + note string + } + var rows []row + mkContainer := func(name string, labels map[string]string) string { + args := []string{"docker", "run", "--detach", "--restart", "no", "--name", ssh.ShellQuote(name)} + for k, v := range labels { + args = append(args, "--label", ssh.ShellQuote(k+"="+v)) + } + args = append(args, ssh.ShellQuote(env.image), "sleep", "300") + return strings.Join(args, " ") + } + var containers []string + t.Cleanup(func() { + cctx, ccancel := context.WithTimeout(context.Background(), 60*time.Second) + defer ccancel() + for _, n := range containers { + probe.Run(cctx, "docker rm -f "+ssh.ShellQuote(n)) + } + for _, suffix := range []string{"a", "b", "c"} { + probe.Run(cctx, "rm -rf "+ssh.ShellQuote(fmt.Sprintf("/deployments/%s-%s", env.app, suffix))) + } + }) + + // ------------------------------------------------------------------ + // (a) Delayed effect completing after owner death. + // ------------------------------------------------------------------ + appA := env.app + "-a" + ownerA := faultConnect(t, env, "owner A") + if err := state.EnsureAppDir(ctx, ownerA, appA); err != nil { + t.Fatalf("(a) creating app dir: %v", err) + } + lkA, err := state.AcquireLockFenced(ctx, ownerA, appA) + if err != nil { + t.Fatalf("(a) owner A acquiring the lock: %v", err) + } + // Owner A issues a long docker effect detached from its session, then + // DIES (session closed, no renewal, no release). The effect — a + // candidate-shaped container of release "deadgen" — lands ~4s later. + lateName := fmt.Sprintf("%s-web-deadgen", appA) + containers = append(containers, lateName) + inner := fmt.Sprintf("sleep 4; docker run --detach --restart no --name %s --label %s --label %s --label %s %s sleep 300", + ssh.ShellQuote(lateName), + ssh.ShellQuote("teploy.app="+appA), + ssh.ShellQuote("teploy.process=web"), + ssh.ShellQuote("teploy.version=deadgen"), + ssh.ShellQuote(env.image)) + if _, err := ownerA.Run(ctx, "nohup sh -c "+ssh.ShellQuote(inner)+" >/dev/null 2>&1 & echo launched"); err != nil { + t.Fatalf("(a) launching the delayed effect: %v", err) + } + ownerA.Close() // owner death: the session dies; the nohup'd effect survives + + // The stale window elapses (simulated by aging the info file — the + // token stays owner A's), then owner B breaks and acquires the lock + // through the REAL stale-break path. + ownerB := faultConnect(t, env, "owner B") + ageLockStale(t, ownerB, appA, lkA.Owner()) + lkB, err := state.AcquireLockFenced(ctx, ownerB, appA) + if err != nil { + t.Fatalf("(a) owner B acquiring the stale-broken lock: %v", err) + } + defer state.ReleaseLockFenced(ownerB, lkB, appA) + + landed := waitRunning(t, ownerB, lateName, 20*time.Second) + if !landed { + t.Errorf("(a) the delayed effect never landed — the fixture's nohup detach did not survive the session; fix the fixture before trusting this scenario") + } + // B reconciles BEFORE acting: it observes the app, then decides. The + // quiescence assumption would be the all-absent observation (RETRY for + // a fresh deploy); the real observation must not collapse to it. + obsA := observe(ctx, ownerB, appA, "newgen", "") + d := Decide(Admitted, obsA) + quiescent := Decide(Admitted, Observation{}) + rows = append(rows, row{ + scenario: "(a) late effect after owner death", + observed: fmt.Sprintf("lock acquired by new owner; foreign running container %s landed post-acquisition=%v", lateName, landed), + decision: fmt.Sprintf("%s (quiescence assumption would say %s)", d, quiescent), + pass: d == Manual && d != quiescent, + note: "acquisition proves nothing; the late effect is unattributable → MANUAL", + }) + + // ------------------------------------------------------------------ + // (b) Crash after side effect, before receipt. + // ------------------------------------------------------------------ + appB := env.app + "-b" + ownerB2 := faultConnect(t, env, "owner B2") + if err := state.EnsureAppDir(ctx, ownerB2, appB); err != nil { + t.Fatalf("(b) creating app dir: %v", err) + } + lkB2, err := state.AcquireLockFenced(ctx, ownerB2, appB) + if err != nil { + t.Fatalf("(b) acquiring the lock: %v", err) + } + defer state.ReleaseLockFenced(ownerB2, lkB2, appB) + candB := fmt.Sprintf("%s-web-c0ffee", appB) + containers = append(containers, candB) + if _, err := ownerB2.Run(ctx, mkContainer(candB, map[string]string{ + "teploy.app": appB, "teploy.process": "web", "teploy.version": "c0ffee", + })); err != nil { + t.Fatalf("(b) starting the candidate side effect: %v", err) + } + obsB := observe(ctx, ownerB2, appB, "c0ffee", "") + dB := Decide(CandidatesRunning, obsB) + rows = append(rows, row{ + scenario: "(b) side effect without receipt", + observed: fmt.Sprintf("candidate %s running; state=%s route=%s record=%s", candB, obsB.StateToCandidate, obsB.RouteToCandidate, obsB.ReleaseRecord), + decision: dB.String(), + pass: dB == Inspect, + note: "reconciliation disposition, never invented success", + }) + + // ------------------------------------------------------------------ + // (c) Abandoned owner: fence break refuses the stale holder's writes. + // ------------------------------------------------------------------ + appC := env.app + "-c" + ownerC1 := faultConnect(t, env, "stale holder C1") + if err := state.EnsureAppDir(ctx, ownerC1, appC); err != nil { + t.Fatalf("(c) creating app dir: %v", err) + } + lkC1, err := state.AcquireLockFenced(ctx, ownerC1, appC) + if err != nil { + t.Fatalf("(c) stale holder acquiring: %v", err) + } + ownerC2 := faultConnect(t, env, "successor C2") + ageLockStale(t, ownerC2, appC, lkC1.Owner()) + lkC2, err := state.AcquireLockFenced(ctx, ownerC2, appC) + if err != nil { + t.Fatalf("(c) successor breaking the stale lock: %v", err) + } + defer state.ReleaseLockFenced(ownerC2, lkC2, appC) + + marker := fmt.Sprintf("/deployments/%s/late-marker", appC) + _, gerr := lkC1.Guarded(ctx, ownerC1, "touch "+ssh.ShellQuote(marker)) + refusedEffect := errors.Is(gerr, state.ErrFenceLost) + markerAbsent := !fileExists(t, ownerC2, marker) + + werr := state.WriteFenced(ctx, ownerC1, appC, state.NewAppliedState(nil, "container", "host", ""), lkC1) + refusedState := errors.Is(werr, state.ErrFenceLost) + stateAbsent := !fileExists(t, ownerC2, fmt.Sprintf("/deployments/%s/state.json", appC)) + rows = append(rows, row{ + scenario: "(c) stale holder's late write", + observed: fmt.Sprintf("guarded effect refused=%v marker-on-disk=%v; fenced state commit refused=%v state.json=%v", + refusedEffect, markerAbsent, refusedState, stateAbsent), + decision: "refused (ErrFenceLost)", + pass: refusedEffect && markerAbsent && refusedState && stateAbsent, + note: "existing fence machinery; no harness-local locking", + }) + + // ------------------------------------------------------------------ + // Result table. + // ------------------------------------------------------------------ + fmt.Println("\n== C01 fault harness — scenario × observed × decision × correctness ==") + fmt.Printf("%-36s | %-72s | %-46s | %s\n", "scenario", "observed", "decision", "correctness") + fmt.Println(strings.Repeat("-", 36) + "-+-" + strings.Repeat("-", 72) + "-+-" + strings.Repeat("-", 46) + "-+-------") + allPass := true + for _, r := range rows { + verdict := "FAIL" + if r.pass { + verdict = "PASS" + } else { + allPass = false + } + fmt.Printf("%-36s | %-72s | %-46s | %s\n", r.scenario, r.observed, r.decision, verdict) + t.Logf("%s — %s (%s)", r.scenario, r.note, verdict) + } + fmt.Println(strings.Repeat("-", 180)) + if !allPass { + t.Error("fault harness: at least one scenario failed — see the table above") + } +} diff --git a/internal/deploy/recovery/recovery.go b/internal/deploy/recovery/recovery.go new file mode 100644 index 0000000..e0ee4df --- /dev/null +++ b/internal/deploy/recovery/recovery.go @@ -0,0 +1,456 @@ +// Package recovery encodes the crash-recovery state table for teploy's +// deploy lifecycle — programme workstream C01, per the implementation +// handoff's "Crash-recovery design obligations": before building the full +// helper/journal, the state transition table (admitted, prepared, +// candidates running, readiness passed, traffic switched, authoritative +// state committed, predecessor retired, terminal receipt persisted) must +// exist as TESTED CODE, with each transition carrying the durable evidence +// that proves it landed and a crash-window disposition: RETRY (safe to +// redo), INSPECT (must reconcile against the target before acting), +// COMPENSATE (undo via known predecessor state), or MANUAL (surface to the +// operator; never auto-decide). +// +// The disposition is a PURE decision function over (from-state, +// observed-target-evidence). It is deliberately evidence-driven: a +// replacement owner that acquired the lock after a stale break must not +// assume lock acquisition proves quiescence — the dead holder's Docker +// effects can still be landing (the integration harness under +// harness_integration_test.go demonstrates exactly that). What the +// replacement owner OBSERVES, cross-checked against what its own records +// say was reached, is the only safe input to a recovery decision. +// +// Evidence names align with what the fenced-lock / releasemeta / Caddy +// machinery persists today, with file:line citations in the ADR +// (docs/C01_RECOVERY_STATE_TABLE.md). The package imports none of the +// effectful packages: it is the decision table, not the reconciler — the +// C01 implementation slices drive effects from these decisions. +package recovery + +// State is one lifecycle state of a deploy attempt. The order is the +// commit order of internal/deploy/deploy.go's DeployFenced; the lattice +// and its evidence citations live in Lattice() and the ADR. +type State uint8 + +const ( + // Admitted: the fenced app lock is held — the owner token names this + // operation in /deployments//.lock/info (internal/state/lock.go:86, + // internal/state/state.go:479). + Admitted State = iota + // Prepared: attempt artifacts are generated and the predecessor + // snapshotted/displaced — /deployments//meta/att/./ + // (internal/releasemeta/attempt.go:107), renames/displacement + // (internal/deploy/deploy.go:414-509). + Prepared + // CandidatesRunning: candidate containers started under exact names + // {app}-{process}-{version}[-{index}] with teploy.* labels + // (internal/docker/docker.go:82-117, internal/deploy/deploy.go:573-607). + CandidatesRunning + // ReadinessPassed: the health gate passed. NO durable receipt exists + // today (internal/deploy/health.go probes are ephemeral) — see ADR + // finding on the readiness receipt. + ReadinessPassed + // TrafficSwitched: the edge routes name the candidates — the managed + // marker block in /deployments/caddy/Caddyfile plus reload + delivery + // verification receipts (internal/caddy/caddy.go:20-48, 584-625, + // 523-550). Host ingress: the candidate holds the fixed port; external + // ingress: no edge step at all. + TrafficSwitched + // AuthoritativeStateCommitted: state.json names the release — the + // fenced rename (internal/state/lock.go:353-381), AppState.Generation + // bumped (internal/state/state.go:68-95). + AuthoritativeStateCommitted + // PredecessorRetired: the snapshotted predecessor workload is + // stopped/removed (internal/deploy/deploy.go:839-861, 963-989). + PredecessorRetired + // TerminalReceiptPersisted: the per-release record + // /deployments//meta/.json (internal/releasemeta/ + // releasemeta.go:216-248) and the /deployments/teploy.log entry + // (internal/state/state.go:662-679) exist. + TerminalReceiptPersisted +) + +// AllStates lists the lifecycle states in commit order. +func AllStates() []State { + return []State{ + Admitted, Prepared, CandidatesRunning, ReadinessPassed, + TrafficSwitched, AuthoritativeStateCommitted, + PredecessorRetired, TerminalReceiptPersisted, + } +} + +func (s State) String() string { + switch s { + case Admitted: + return "admitted" + case Prepared: + return "prepared" + case CandidatesRunning: + return "candidates-running" + case ReadinessPassed: + return "readiness-passed" + case TrafficSwitched: + return "traffic-switched" + case AuthoritativeStateCommitted: + return "authoritative-state-committed" + case PredecessorRetired: + return "predecessor-retired" + case TerminalReceiptPersisted: + return "terminal-receipt-persisted" + default: + return "unknown-state" + } +} + +// Disposition is the crash-window disposition of a recovery decision. +type Disposition uint8 + +const ( + // Retry: safe to redo. The transition's effects are idempotent, never + // landed, or the remaining tail converges (record/backfill semantics). + Retry Disposition = iota + // Inspect: must reconcile against the target before acting. Evidence + // is readable but does not match any single crash point (or is + // unreadable-but-retryable, like a transient inventory failure) — the + // owner re-observes and re-decides; it never treats a side effect as + // a committed success. + Inspect + // Compensate: undo via known predecessor state. The predecessor is + // running or restorable (stopped/displaced), so the partial effect can + // be rolled back to a recorded, serving generation. + Compensate + // Manual: surface to the operator; never auto-decide. Unattributable + // containers, unreadable authoritative bookkeeping, or a dark app with + // no compensable predecessor. + Manual +) + +func (d Disposition) String() string { + switch d { + case Retry: + return "RETRY" + case Inspect: + return "INSPECT" + case Compensate: + return "COMPENSATE" + case Manual: + return "MANUAL" + default: + return "unknown-disposition" + } +} + +// Evidence is the tri-state of one observation class. Absent and Present +// are PROVEN states of the target (confirmed by docker inspect / file +// read), never guesses: names and receipts are exact, per the handoff. +// Unknown is unreadable, unverifiable, or conflicting. +type Evidence uint8 + +const ( + Absent Evidence = iota // provably not present + Present // provably present + Unknown // unreadable / unverifiable / conflicting +) + +func (e Evidence) String() string { + switch e { + case Absent: + return "absent" + case Present: + return "present" + default: + return "unknown" + } +} + +// Observation is what a recovery owner can observe about the target after +// acquiring the app lock. Every field is evidence about the ATTEMPTED +// release (the release the crashed operation was deploying) versus its +// known predecessor — the two identities the decision table reasons over. +// Collection of an Observation from a live host is the harness's job +// (harness_integration_test.go); the decision over it is pure. +type Observation struct { + // Candidates: a RUNNING workload under the exact candidate names of + // the attempted release ({app}-{process}-{version}[-{index}], labels + // teploy.app/process/version — internal/docker/docker.go:82-117). + Candidates Evidence + // CandidateCorpses: candidate-named containers exist but are NOT + // running (created-but-unstarted, exited) — the reconcilePartialRun + // class (internal/deploy/deploy.go:1282-1292). + CandidateCorpses Evidence + // ForeignCandidates: RUNNING containers labeled teploy.app= whose + // names match NEITHER the attempted release's candidate names NOR the + // predecessor's (a late effect from a dead owner landing under a name + // the new owner never minted, an orphaned generation, a hand-run + // container). Unattributable by construction — always MANUAL. + ForeignCandidates Evidence + // RouteToCandidate: the managed Caddy block / LB upstreams name the + // attempted release's candidate containers (marker block + // "# TEPLOY BEGIN " — internal/caddy/caddy.go:20-21, 584-625). + // Host ingress: the fixed port is held by the candidate. External + // ingress: always Absent (no teploy-managed edge). + RouteToCandidate Evidence + // RouteToPredecessor: the managed block names the predecessor + // generation's containers (or no managed block exists — Absent + // together with RouteToCandidate means "no edge step", which is + // normal for external ingress and a first deploy). + RouteToPredecessor Evidence + // StateToCandidate: state.json current_hash names the attempted + // release — the fenced commit landed (internal/state/lock.go:353-381). + StateToCandidate Evidence + // StateToPredecessor: state.json names the predecessor release. Both + // state fields Absent = no state file at all (first deploy, or wiped). + StateToPredecessor Evidence + // PredecessorServing: the predecessor workload (including a + // same-version _replaced rename) is RUNNING — a live, known generation + // to compensate back to. + PredecessorServing Evidence + // PredecessorStopped: predecessor containers exist but are stopped + // (displaced to free a fixed port, or retirement half-done) — + // restorable by restart, so still a compensation source. + PredecessorStopped Evidence + // ReleaseRecord: the per-release record + // /deployments//meta/.json exists for the attempted + // release (internal/releasemeta/releasemeta.go:168-176). Informational + // for the decision: records converge (same-version rewrite, backfill), + // so presence/absence never selects a disposition by itself. + ReleaseRecord Evidence +} + +// Decide is the transition table's crash-window decision function: given +// the state the recovery owner's own records say was reached, and the +// evidence observed on the target, what may automation do? +// +// Rules, in evaluation order (each is justified in the ADR): +// +// R0 Encoding-impossible evidence (one state.json naming two releases, +// one predecessor both running and stopped) → MANUAL: never act on +// bookkeeping that contradicts itself. +// R1 Unattributable workloads (ForeignCandidates present) → MANUAL. +// R2 The authority is unreadable (either state field Unknown) → MANUAL: +// without state.json no action can know whether it is redoing, +// undoing, or hijacking. +// R3 Any other unreadable class → INSPECT: re-observe (transient +// inventory/parse failures are reconcile triggers, not operator +// escalations), but never RETRY or COMPENSATE on unreadable evidence. +// R3 Traffic PROVEN on a generation the authority does not name, with +// the predecessor PROVEN gone (not running, not stopped-restorable) +// → MANUAL, regardless of what the records claim (keep-or-rebuild is +// an operator decision, and a record that contradicts the readable +// authority is a reason NOT to act). Evaluated before the unreadable +// classes: when the dark-window fact is already proven, unreadable +// side evidence cannot make it actionable. +// R4 Any other unreadable class → INSPECT: re-observe (transient +// inventory/parse failures are reconcile triggers, not operator +// escalations), but never RETRY or COMPENSATE on unreadable evidence. +// R5 Conflicting edge evidence (route names BOTH generations) → INSPECT: +// reconcile against the records (ParseSites/ExtractPolicy exist for +// exactly this — internal/caddy/routes.go:89,429). +// R6 Record/target disagreement (records say the commit was reached but +// state does not name the release, or vice versa) → INSPECT: the +// crash happened at a different point than recorded, or an operator +// moved authority (rollback rewrites state.json) — reconcile which +// generation is real before ANY action, even a redo that looks safe. +// R7 Authority dispatch: +// - state names the ATTEMPTED release (committed): +// edge still serving the predecessor → INSPECT; otherwise RETRY — +// the tail (retirement, receipts) is idempotent/convergent. +// - state names the PREDECESSOR (uncommitted): +// edge on the candidates → COMPENSATE (the no-predecessor case +// was settled by R3); +// candidates running, edge unchanged → INSPECT (side effect +// landed, outcome unreconciled — never invented success); +// displaced/stopped predecessor with no live candidate → +// COMPENSATE (restart it; the app is dark in the recreate window); +// otherwise → RETRY (the effect never landed). +// - no state at all: candidates running → INSPECT; otherwise RETRY +// (clean first deploy; the edge-on-candidates case was settled by +// R3 — MANUAL). +// +// Decide is pure and total: every (State, Observation) yields a decision. +// The exhaustive test enumerates the full product space. +func Decide(from State, o Observation) Disposition { + // R0: self-contradictory bookkeeping. + if o.StateToCandidate == Present && o.StateToPredecessor == Present { + return Manual + } + if o.PredecessorServing == Present && o.PredecessorStopped == Present { + return Manual + } + // R1: unattributable workloads are the handoff's explicit conflict + // class ("unknown container names") — never auto-decide. + if o.ForeignCandidates == Present { + return Manual + } + // R2: the authority is unreadable. + if o.StateToCandidate == Unknown || o.StateToPredecessor == Unknown { + return Manual + } + // R3: the proven-dark window. Traffic provably sits on a generation + // the authority does not name, and the predecessor is provably not + // restorable — keep-or-rebuild is an operator decision, whatever the + // recovery owner's own records claim. + if o.RouteToCandidate == Present && o.StateToCandidate == Absent && + o.PredecessorServing == Absent && o.PredecessorStopped == Absent { + return Manual + } + // R4: any other unreadable class is a reconcile trigger, never a + // license to redo or undo. + if o.Candidates == Unknown || o.CandidateCorpses == Unknown || + o.ForeignCandidates == Unknown || + o.RouteToCandidate == Unknown || o.RouteToPredecessor == Unknown || + o.PredecessorServing == Unknown || o.PredecessorStopped == Unknown { + return Inspect + } + // R5: the edge names both generations at once. + if o.RouteToCandidate == Present && o.RouteToPredecessor == Present { + return Inspect + } + // R6: the recorded state and the target's authority disagree on + // whether the commit happened. + recordedCommitted := from >= AuthoritativeStateCommitted + observedCommitted := o.StateToCandidate == Present + if recordedCommitted != observedCommitted { + return Inspect + } + + // R7: authority dispatch. + switch { + case o.StateToCandidate == Present: + // Committed: the attempted release IS authoritative. Anything left + // is tail convergence (retirement redo, record write/backfill, + // log) — except an edge still serving the predecessor generation, + // which is an authority/edge contradiction to reconcile. + if o.RouteToPredecessor == Present { + return Inspect + } + return Retry + + case o.StateToPredecessor == Present: + // Uncommitted: dispatch on where partial effects stand. + if o.RouteToCandidate == Present { + // Traffic is on an uncommitted generation — the classic + // traffic-switched crash window (abortStateCommit's shape). + // The no-predecessor case was settled by R3 (MANUAL); a + // restorable predecessor makes this COMPENSATE. + return Compensate + } + if o.Candidates == Present { + // Side effect landed, no receipt: reconcile — the handoff's + // decisive "never invented success" case. + return Inspect + } + if o.PredecessorStopped == Present { + // No live candidate and the predecessor is displaced/stopped: + // the recreate-strategy window — the app is dark until the + // predecessor comes back. + return Compensate + } + // Nothing landed (or only corpses under candidate names, which the + // next attempt reconciles) and the predecessor is serving or never + // existed: safe to redo. + return Retry + + default: + // No state file at all: a first deploy (or wiped bookkeeping). + // Route-on-candidate with no restorable predecessor was settled by + // R3 (MANUAL); a running candidate is a side effect to + // reconcile; otherwise this is a clean first deploy. + if o.Candidates == Present { + return Inspect + } + return Retry + } +} + +// Transition is one edge of the deploy lifecycle lattice: the durable +// evidence that proves it landed, and the crash-window disposition for an +// owner that dies inside the transition's window (after the From state's +// effects began, before the To state's receipt exists). +type Transition struct { + From State + To State + // Evidence names the durable receipts proving the transition landed, + // aligned with what the fenced-lock / releasemeta / Caddy code + // persists today (file:line citations below and in the ADR). + Evidence []string + // CrashDisposition is the table's disposition for a crash inside this + // transition's window when the observed evidence matches the window's + // canonical partial state. + CrashDisposition Disposition + // Note records ingress-mode variants and where the current code's + // machinery cannot yet produce or consume the evidence (the ADR's + // disagreement findings). + Note string +} + +// Lattice returns the forward transition table. The crash dispositions +// here are the canonical per-window answers; Decide generalizes them over +// arbitrary observed evidence (and the exhaustive test proves the two +// agree on canonical observations). +func Lattice() []Transition { + return []Transition{ + { + From: Admitted, To: Prepared, + Evidence: []string{ + "attempt dir /deployments//meta/att/./ (releasemeta/attempt.go:107-116)", + "lock owner token in /deployments//.lock/info (state/lock.go:86-92, state.go:522-533)", + }, + CrashDisposition: Retry, + Note: "retryable: artifacts are attempt-scoped (random id, write-once) and a fresh attempt collides with nothing", + }, + { + From: Prepared, To: CandidatesRunning, + Evidence: []string{ + "container IDs returned by docker run (deploy/deploy.go:577-607)", + "exact names {app}-{process}-{version}[-{index}] + labels teploy.app/process/version (docker/docker.go:82-117,160-165)", + }, + CrashDisposition: Inspect, + Note: "inspect: a running candidate with no receipt is never success; corpses under candidate names are reconciled by the next attempt (deploy.go:1282-1292). Disagreeing today: names are version-keyed, so two attempts of one hash are not attributable (ADR finding; register F04/A09)", + }, + { + From: CandidatesRunning, To: ReadinessPassed, + Evidence: []string{ + "NONE DURABLE — health-gate results are ephemeral (deploy/health.go)", + }, + CrashDisposition: Inspect, + Note: "the only transition with no receipt today; a recovery owner must re-probe. ADR finding: readiness receipt is a design obligation", + }, + { + From: ReadinessPassed, To: TrafficSwitched, + Evidence: []string{ + "managed marker block '# TEPLOY BEGIN '..'END' naming candidate upstreams in /deployments/caddy/Caddyfile (caddy/caddy.go:20-21,584-625)", + "reload receipt: docker exec caddy caddy reload (caddy/caddy.go:29-32)", + "delivery verification md5 host-vs-container (caddy/caddy.go:523-550)", + }, + CrashDisposition: Compensate, + Note: "compensate via the recorded/serving predecessor (abortStateCommit's shape, deploy.go:1036-1095); MANUAL when the predecessor is gone. ingress:host variant: candidate holds the fixed port; ingress:external: no edge step (transition is a no-op)", + }, + { + From: TrafficSwitched, To: AuthoritativeStateCommitted, + Evidence: []string{ + "fenced rename of /deployments//state.json naming the release (state/lock.go:353-381)", + "AppState.Generation / OperationID (state/state.go:68-95)", + }, + CrashDisposition: Compensate, + Note: "the commit is the single fenced atomic effect; a crash before it leaves traffic on an uncommitted generation", + }, + { + From: AuthoritativeStateCommitted, To: PredecessorRetired, + Evidence: []string{ + "predecessor snapshot containers stopped (deploy/deploy.go:839-861,963-989)", + "absence of predecessor names in docker ps label inventory (docker/docker.go:568-571)", + }, + CrashDisposition: Retry, + Note: "retryable: retirement re-derives from the inventory; failures are reported, never silent. ADR finding: the success log entry records no degraded flag", + }, + { + From: PredecessorRetired, To: TerminalReceiptPersisted, + Evidence: []string{ + "record /deployments//meta/.json, 0600, atomic (releasemeta/releasemeta.go:216-248)", + "log entry appended to /deployments/teploy.log (state/state.go:662-679)", + }, + CrashDisposition: Retry, + Note: "retryable/convergent: same-version record rewrite and live-container backfill (releasemeta.go:336-473) both heal a missing record. ADR finding: nothing reconciles it until the next deploy", + }, + } +} diff --git a/internal/deploy/recovery/recovery_test.go b/internal/deploy/recovery/recovery_test.go new file mode 100644 index 0000000..216913a --- /dev/null +++ b/internal/deploy/recovery/recovery_test.go @@ -0,0 +1,423 @@ +package recovery + +import ( + "fmt" + "testing" +) + +// The tests are the point (the handoff explicitly rejects a doc-only +// table): the canonical table pins each state's crash-window disposition, +// the exhaustive product-space test pins the safety invariants over EVERY +// (state, observation) pair, and the conflict scenarios pin the handoff's +// named evidence conflicts to exact dispositions. + +// quiescentPredecessorWorld is the canonical observation of an app whose +// authority names the predecessor, whose edge serves the predecessor, and +// whose predecessor is running — the "nothing of the attempt landed" +// world. +func quiescentPredecessorWorld() Observation { + return Observation{ + Candidates: Absent, + CandidateCorpses: Absent, + ForeignCandidates: Absent, + RouteToCandidate: Absent, + RouteToPredecessor: Present, + StateToCandidate: Absent, + StateToPredecessor: Present, + PredecessorServing: Present, + PredecessorStopped: Absent, + ReleaseRecord: Absent, + } +} + +// quiescentFirstDeploy is the canonical no-predecessor world: no state, no +// edge, nothing running. +func quiescentFirstDeploy() Observation { + return Observation{ + Candidates: Absent, + CandidateCorpses: Absent, + ForeignCandidates: Absent, + RouteToCandidate: Absent, + RouteToPredecessor: Absent, + StateToCandidate: Absent, + StateToPredecessor: Absent, + PredecessorServing: Absent, + PredecessorStopped: Absent, + ReleaseRecord: Absent, + } +} + +// candidateSideEffectNoReceipt is the handoff's scenario (b): candidate +// containers RUNNING, edge and authority untouched, no release record. +func candidateSideEffectNoReceipt() Observation { + o := quiescentPredecessorWorld() + o.Candidates = Present + o.ReleaseRecord = Absent + return o +} + +// trafficOnUncommitted is the traffic-switched crash window: edge names +// the candidates, authority still names the predecessor, predecessor +// serving. +func trafficOnUncommitted() Observation { + o := candidateSideEffectNoReceipt() + o.RouteToCandidate = Present + o.RouteToPredecessor = Absent + return o +} + +// committedWorld is the post-commit world: authority names the attempted +// release, edge follows it, candidates running, predecessor still +// serving (its retirement is the remaining tail). +func committedWorld() Observation { + return Observation{ + Candidates: Present, + CandidateCorpses: Absent, + ForeignCandidates: Absent, + RouteToCandidate: Present, + RouteToPredecessor: Absent, + StateToCandidate: Present, + StateToPredecessor: Absent, + PredecessorServing: Present, + PredecessorStopped: Absent, + ReleaseRecord: Absent, + } +} + +// TestCanonicalTable pins each state's canonical crash-window disposition: +// the decision an owner gets when the target looks EXACTLY like the +// canonical partial state of a crash just inside that state's window. +// These must match Lattice()'s CrashDisposition for the transition LEAVING +// each state. +func TestCanonicalTable(t *testing.T) { + cases := []struct { + from State + obs Observation + want Disposition + why string + }{ + // Admitted: crash right after acquiring the lock. Blue/green world + // (predecessor serving) and first deploy both retry — nothing of + // the attempt landed. + {Admitted, quiescentPredecessorWorld(), Retry, "nothing landed, predecessor serving"}, + {Admitted, quiescentFirstDeploy(), Retry, "nothing landed, first deploy"}, + // Prepared: artifacts exist. Blue/green retries (the predecessor + // was never touched); the recreate-strategy window (predecessor + // DISPLACED to free the fixed port, no candidate yet) is dark and + // must be compensated (restart the displaced workload). + {Prepared, quiescentPredecessorWorld(), Retry, "artifacts only, predecessor serving"}, + {Prepared, func() Observation { + o := quiescentPredecessorWorld() + o.PredecessorServing = Absent + o.PredecessorStopped = Present + return o + }(), Compensate, "recreate window: predecessor displaced, app dark"}, + // CandidatesRunning: containers started, readiness not passed, no + // receipts. Never success; reconcile. + {CandidatesRunning, candidateSideEffectNoReceipt(), Inspect, "side effect landed, no receipt"}, + {CandidatesRunning, quiescentPredecessorWorld(), Retry, "containers never landed (late docker run still pending would show as foreign/unknown — this row is the provably-empty case)"}, + // ReadinessPassed: the readiness gate has NO durable receipt, so + // its canonical world is indistinguishable from CandidatesRunning's + // — same disposition (ADR finding: the missing receipt collapses + // the two windows). + {ReadinessPassed, candidateSideEffectNoReceipt(), Inspect, "readiness is unobservable post-crash; reconcile"}, + // TrafficSwitched: edge on the uncommitted generation. + {TrafficSwitched, trafficOnUncommitted(), Compensate, "traffic on uncommitted generation, predecessor restorable"}, + {TrafficSwitched, func() Observation { + o := trafficOnUncommitted() + o.PredecessorServing = Absent + o.PredecessorStopped = Present + return o + }(), Compensate, "predecessor displaced but restorable"}, + {TrafficSwitched, func() Observation { + o := trafficOnUncommitted() + o.PredecessorServing = Absent + o.PredecessorStopped = Absent + return o + }(), Manual, "predecessor gone: keep-or-rebuild is an operator decision"}, + // AuthoritativeStateCommitted: finish the tail (retirement, + // receipts) — idempotent. + {AuthoritativeStateCommitted, committedWorld(), Retry, "committed; tail converges"}, + {AuthoritativeStateCommitted, func() Observation { + o := committedWorld() + o.PredecessorServing = Absent + o.PredecessorStopped = Absent + o.ReleaseRecord = Absent + return o + }(), Retry, "committed; predecessor already gone, record still converges"}, + // PredecessorRetired: persist the receipts. + {PredecessorRetired, func() Observation { + o := committedWorld() + o.PredecessorServing = Absent + o.PredecessorStopped = Absent + return o + }(), Retry, "retired; write/verify receipts"}, + // TerminalReceiptPersisted: converged; RETRY here means verify-only + // reconvergence (the redo does nothing). + {TerminalReceiptPersisted, func() Observation { + o := committedWorld() + o.PredecessorServing = Absent + o.PredecessorStopped = Absent + o.ReleaseRecord = Present + return o + }(), Retry, "converged; verify-only"}, + } + for _, tc := range cases { + got := Decide(tc.from, tc.obs) + if got != tc.want { + t.Errorf("Decide(%s, %s) = %s, want %s (%s)", tc.from, tc.why, got, tc.want, tc.why) + } + } +} + +// TestLatticeMatchesCanonicalWindows proves the data lattice and the +// decision function agree: for each forward transition, crashing inside +// its window (the To-state's effects may have begun, its receipt does not +// exist) with the canonical partial evidence yields the lattice's +// recorded CrashDisposition. +func TestLatticeMatchesCanonicalWindows(t *testing.T) { + for _, tr := range Lattice() { + var obs Observation + switch tr.To { + case Prepared: + // Artifact generation window: candidates never started. + obs = quiescentPredecessorWorld() + case CandidatesRunning, ReadinessPassed: + // Candidates may be landing/landed; readiness leaves no + // receipt, so both windows share the side-effect world. + obs = candidateSideEffectNoReceipt() + case TrafficSwitched, AuthoritativeStateCommitted: + // Edge may have switched; the state commit has not landed. + obs = trafficOnUncommitted() + case PredecessorRetired: + obs = committedWorld() + case TerminalReceiptPersisted: + obs = committedWorld() + obs.PredecessorServing = Absent + obs.ReleaseRecord = Absent + } + if got := Decide(tr.From, obs); got != tr.CrashDisposition { + t.Errorf("lattice says %s for %s→%s, Decide says %s", tr.CrashDisposition, tr.From, tr.To, got) + } + } +} + +// TestHandoffConflictScenarios pins the handoff's named conflicting +// evidence classes to exact dispositions, from every state. +func TestHandoffConflictScenarios(t *testing.T) { + // "Candidate container running but route never switched": INSPECT from + // every state whose record is pre-commit; from post-commit records the + // record/target disagreement rule also yields INSPECT — i.e. a running + // uncommitted side effect is INSPECT no matter what the record claims. + o := candidateSideEffectNoReceipt() + for _, from := range AllStates() { + if got := Decide(from, o); got != Inspect { + t.Errorf("running candidate + untouched route from %s = %s, want INSPECT (never invented success)", from, got) + } + } + + // "Unknown container names" (a running app-labeled container neither + // candidate nor predecessor): MANUAL from every state. + o = quiescentPredecessorWorld() + o.ForeignCandidates = Present + for _, from := range AllStates() { + if got := Decide(from, o); got != Manual { + t.Errorf("foreign candidate from %s = %s, want MANUAL (unattributable)", from, got) + } + } + + // "Predecessor already retired" while traffic sits on the uncommitted + // generation: MANUAL — there is nothing recorded to compensate to. + o = trafficOnUncommitted() + o.PredecessorServing = Absent + for _, from := range AllStates() { + if got := Decide(from, o); got != Manual { + t.Errorf("traffic on uncommitted generation + predecessor gone from %s = %s, want MANUAL", from, got) + } + } + + // Crash after side effect before receipt, first deploy (no state at + // all): INSPECT from every state — never success, never a blind retry. + o = Observation{Candidates: Present} + for _, from := range AllStates() { + if got := Decide(from, o); got != Inspect { + t.Errorf("candidate running with NO state file from %s = %s, want INSPECT", from, got) + } + } + + // A route naming BOTH generations (mixed upstreams / split evidence): + // INSPECT — reconcile against the records. + o = trafficOnUncommitted() + o.RouteToPredecessor = Present + for _, from := range AllStates() { + if got := Decide(from, o); got != Inspect { + t.Errorf("route naming both generations from %s = %s, want INSPECT", from, got) + } + } + + // An operator rolled back between the record and the recovery + // (authority moved back to the predecessor while the recorded state + // says committed): INSPECT — never a blind redeploy that would undo + // the operator's decision. + o = quiescentPredecessorWorld() + for _, from := range []State{AuthoritativeStateCommitted, PredecessorRetired, TerminalReceiptPersisted} { + if got := Decide(from, o); got != Inspect { + t.Errorf("committed record + predecessor authority from %s = %s, want INSPECT (record/target disagreement)", from, got) + } + } +} + +// TestExhaustiveProductSpace enumerates EVERY (state, observation) pair — +// 8 states × 3^10 evidence combinations — and asserts the safety +// invariants. A doc table cannot do this; this is the contract the C01 +// implementation slices build on. +func TestExhaustiveProductSpace(t *testing.T) { + states := AllStates() + fields := 10 + total := 1 + for range fields { + total *= 3 + } + checked := 0 + for _, from := range states { + for combo := 0; combo < total; combo++ { + var o Observation + decode(combo, &o) + d := Decide(from, o) + checkInvariants(t, from, o, d) + checked++ + } + } + if checked != len(states)*total { + t.Fatalf("enumerated %d pairs, expected %d", checked, len(states)*total) + } +} + +// decode fills o from a base-3 encoding of its ten Evidence fields. +func decode(combo int, o *Observation) { + vals := []Evidence{Absent, Present, Unknown} + fs := []*Evidence{ + &o.Candidates, &o.CandidateCorpses, &o.ForeignCandidates, + &o.RouteToCandidate, &o.RouteToPredecessor, + &o.StateToCandidate, &o.StateToPredecessor, + &o.PredecessorServing, &o.PredecessorStopped, &o.ReleaseRecord, + } + for _, f := range fs { + *f = vals[combo%3] + combo /= 3 + } +} + +// checkInvariants asserts the cross-cutting safety contract for one +// (state, observation, decision) triple. Every rule here is a property +// the ADR's disposition rules promise for ALL inputs. +func checkInvariants(t *testing.T, from State, o Observation, d Disposition) { + t.Helper() + desc := fmt.Sprintf("Decide(%s, %+v) = %s", from, o, d) + + // I1: unattributable workloads are never auto-decided. + if o.ForeignCandidates == Present && d != Manual { + t.Errorf("%s: foreign candidate must be MANUAL", desc) + } + // I2: unreadable authority is never auto-decided. + if (o.StateToCandidate == Unknown || o.StateToPredecessor == Unknown) && d != Manual { + t.Errorf("%s: unreadable state must be MANUAL", desc) + } + // I3: no redo or undo on unreadable non-authority evidence. + if d == Retry || d == Compensate { + for _, e := range []Evidence{o.Candidates, o.CandidateCorpses, o.ForeignCandidates, + o.RouteToCandidate, o.RouteToPredecessor, + o.PredecessorServing, o.PredecessorStopped} { + if e == Unknown { + t.Errorf("%s: RETRY/COMPENSATE on unreadable evidence", desc) + } + } + } + // I4: compensation always has a restorable predecessor and an + // uncommitted authority — undo targets the recorded/serving + // predecessor generation, never a guess. + if d == Compensate && + !(o.StateToPredecessor == Present && (o.PredecessorServing == Present || o.PredecessorStopped == Present)) { + t.Errorf("%s: COMPENSATE without a restorable predecessor under predecessor authority", desc) + } + // I5: retry requires fully readable, non-conflicting, attributable + // evidence — including self-consistent authority encoding. + if d == Retry { + if o.ForeignCandidates == Present { + t.Errorf("%s: RETRY with foreign candidate", desc) + } + if o.RouteToCandidate == Present && o.RouteToPredecessor == Present { + t.Errorf("%s: RETRY with conflicting route evidence", desc) + } + if o.StateToCandidate == Present && o.StateToPredecessor == Present { + t.Errorf("%s: RETRY with impossible state encoding", desc) + } + if o.PredecessorServing == Present && o.PredecessorStopped == Present { + t.Errorf("%s: RETRY with impossible predecessor encoding", desc) + } + } + // I6: traffic on the uncommitted generation with the predecessor gone + // is never auto-decided (keep-or-rebuild is an operator call). + if o.RouteToCandidate == Present && o.StateToPredecessor == Present && + o.PredecessorServing == Absent && o.PredecessorStopped == Absent && + o.StateToCandidate == Absent && o.ForeignCandidates == Absent && + d != Manual { + t.Errorf("%s: uncommitted traffic + gone predecessor must be MANUAL", desc) + } + // I7: never invented success — a running candidate under uncommitted + // or absent authority, with the edge NOT committed to it either, is + // INSPECT at minimum (Inspect or Manual). When the edge IS on the + // candidates the world is the explicit traffic-switched window and + // COMPENSATE is the designed answer, so it is exempt. + if o.Candidates == Present && o.StateToCandidate != Present && + o.RouteToCandidate != Present && + o.ForeignCandidates == Absent && + o.StateToCandidate != Unknown && o.StateToPredecessor != Unknown && + d != Inspect && d != Manual { + t.Errorf("%s: uncommitted running candidate must INSPECT or MANUAL, got %s", desc, d) + } +} + +// TestExhaustiveCanonicalCoverage spot-proves the enumeration actually +// reaches the canonical worlds (guards a decode bug vacuously passing the +// invariant checks). +func TestExhaustiveCanonicalCoverage(t *testing.T) { + // foreignWorld: a running app-labeled container under unknown names. + foreignWorld := quiescentPredecessorWorld() + foreignWorld.ForeignCandidates = Present + // darkWorld: traffic on the uncommitted generation, predecessor gone. + darkWorld := trafficOnUncommitted() + darkWorld.PredecessorServing = Absent + + seen := map[Disposition]bool{} + for _, from := range AllStates() { + for _, o := range []Observation{ + quiescentPredecessorWorld(), quiescentFirstDeploy(), + candidateSideEffectNoReceipt(), trafficOnUncommitted(), + committedWorld(), foreignWorld, darkWorld, + } { + // Prove decode() can express it: find its combo and re-decode. + if !comboExists(o) { + t.Fatalf("decode space does not express %+v", o) + } + seen[Decide(from, o)] = true + } + } + for _, d := range []Disposition{Retry, Inspect, Compensate, Manual} { + if !seen[d] { + t.Errorf("disposition %s never produced by canonical worlds", d) + } + } +} + +func comboExists(o Observation) bool { + total := 59049 // 3^10 + for combo := 0; combo < total; combo++ { + var probe Observation + decode(combo, &probe) + if probe == o { + return true + } + } + return false +} From 28d130e11192ed4902367ff7c06aef08749239a1 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:52:17 -0700 Subject: [PATCH 04/12] =?UTF-8?q?docs(audit):=20C01=20fault=20harness=20ex?= =?UTF-8?q?ecuted=20against=20real=20SSH+Docker=20fixture=20=E2=80=94=20al?= =?UTF-8?q?l=20scenarios=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AUDIT_OPEN.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index c2262f1..d4b6484 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -950,3 +950,17 @@ clean; `go vet -tags integration ./internal/deploy/recovery` clean; `go test ./... -race` all packages ok (integration-tagged code excluded by default); `go test -tags integration …TestFaultHarness` skips cleanly with env unset; gofmt clean. + +### C01 spike — executed against a real target (2026-09-21, later) + +The fault harness ran against a real SSH+Docker fixture (colima VM, +Linux aarch64, Docker server 29.5.2, /deployments provisioned): scenario +(a) late effect after owner death — new owner acquired the lock, the +dead owner's container landed AFTER acquisition, decision = MANUAL +("lock acquisition proves quiescence" would have said RETRY); (b) side +effect without receipt — running candidate, no state/route/record, +decision = INSPECT, never invented success; (c) stale holder's late +write — the EXISTING fence machinery refused both the guarded effect +and the fenced state commit (ErrFenceLost). Result table preserved in +the session receipt. The design spike's executable-proof obligation is +met; the C01-1..C01-10 disagreement implementations remain open. From f52aec769c2708443394bb1e8cf6d1383f6e90d4 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:15:34 -0700 Subject: [PATCH 05/12] docs: changelog v0.1.35 (compose contracts, recovery table) + v0.1.34 backfill --- CHANGELOG.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bff783d..8050fbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,74 @@ All notable changes to teploy are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [0.1.35] - 2026-09-22 + +### Fixed + +- **Compose import no longer loses the application port.** `ports: + ['8080:3000']` means container port 3000 bound to host 8080; the + importer previously used ports only to pick the web service and + discarded them, so the app deployed as `:80`, failed its health check + and rolled back. The container port now imports as the application + port (with bare ports, IP-prefixed bindings and IPv6 forms supported; + non-TCP entries preserved into `publish`), and unsupported grammar — + ranges, long-form port objects, multiple distinct container ports, + UDP-only services — is refused naming the service and the reason + instead of being silently reinterpreted. +- **Compose import refuses services it cannot faithfully deploy.** + Previously a service built from a different context than the app's + (`jobs: build ./jobs` next to `web: build ./web`) was silently + flattened into a process of the app's image: the wrong code ran under + the right command and the import reported success. It is now refused + naming the service, its build context and the remediation (shared + context, prebuilt image, or teploy.yml). Same-build workers are + unaffected. This is a deliberate behavior change: files that imported + "successfully" while deploying something other than what they declared + now fail fast at import time. +- **Compose fields are classified instead of silently ignored.** The + importer's non-strict YAML parse accepted files using `healthcheck`, + `networks`, `secrets`, `configs`, `profiles`, `deploy`, `env_file`, + `entrypoint` and security options while dropping their semantics. + Now: service healthchecks translate to teploy's `health:` block, + no-op values are tolerated, non-default profiles skip the service + (matching `docker compose up` semantics), metadata is ignored with + reasons, and everything with semantics teploy cannot preserve is + rejected naming the service and field. Another deliberate behavior + change in the same spirit as above. + +### Added + +- **Crash-recovery state table for the deploy lifecycle** (design spike + for the transaction work): an eight-state lattice with an exhaustive, + property-tested disposition function (retry / inspect / compensate / + manual) over crash evidence, an ADR mapping it onto the existing + fenced-lock and release-record machinery, and an integration-tagged + fault harness (`go test -tags integration ./internal/deploy/recovery` + with `TEPLOY_FAULT_*` env) that drives a real SSH+Docker host through + late-effect-after-owner-death, side-effect-without-receipt and + stale-holder scenarios. No deploy behavior changed in this release by + this table; it is the specification the recovery work implements + against. + +## [0.1.34] - 2026-09-11 + +### Fixed + +- Template rendering is YAML-safe with path-keyed generated secrets and + a bounded, validated registry fetch; the templates corpus test renders + the whole catalog. +- Homebrew formulas carry a version test block. +- Deterministic Compose import (sorted web-service selection) and + exec-form command quoting that survives the container's `sh -c` + re-parse. +- Registry-port image references parse correctly in backup/restore, and + `.env` is archived/restored at its app-level location. +- Scheduled backups: only the archive just created is uploaded, + `--endpoint` applies to every aws call, `%` escapes survive crond, and + accessory restore uses a per-invocation temp directory; MySQL + dump/restore passes the root password via `MYSQL_PWD` instead of + argv. + ## [0.1.33] - 2026-09-01 ### Fixed From 127fe1d9bfd652b65469ba80ac161a6ddb6715fe Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:31:24 -0700 Subject: [PATCH 06/12] Allow absolute-path volume keys as host binds in teploy.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docker layer already treats volume keys as host paths (RunConfig documents host_path -> container_path), but config validation only accepted the name grammar, so the documented preview/delivery pattern (mount a trusted clone + credentials into a worker) could not be expressed in teploy.yml at all — only by hand-rolled docker -v outside the manifest. Named volumes resolve under /deployments//volumes and are teploy-created; a bind mounts what the operator already owns, exactly as given, and teploy never creates or relocates it. Destinations must now be absolute (optionally :ro/:rw), on both deploy paths. Found from teploy-ship's S14 trusted-copy provisioning. --- internal/cli/deploy.go | 7 ++++++ internal/cli/singledeploy.go | 7 +++++- internal/config/app.go | 36 ++++++++++++++++++++++++++++--- internal/config/hardening_test.go | 33 ++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index 32e964a..0974d2d 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -581,6 +581,13 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * if len(appCfg.Volumes) > 0 { volumes = make(map[string]string, len(appCfg.Volumes)) for name, containerPath := range appCfg.Volumes { + // A host bind mounts a directory the operator owns, exactly as + // given — teploy never creates or relocates it (it may hold a + // clone with credentials, or anything else that is not app data). + if config.IsHostBindVolume(name) { + volumes[name] = containerPath + continue + } hostPath := fmt.Sprintf("/deployments/%s/volumes/%s", appCfg.App, name) volumes[hostPath] = containerPath if _, err := executor.Run(ctx, fmt.Sprintf("mkdir -p %s", hostPath)); err != nil { diff --git a/internal/cli/singledeploy.go b/internal/cli/singledeploy.go index 30a2690..d6c9be1 100644 --- a/internal/cli/singledeploy.go +++ b/internal/cli/singledeploy.go @@ -186,9 +186,14 @@ func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.App if len(appCfg.Volumes) > 0 { volumes = make(map[string]string, len(appCfg.Volumes)) for name, containerPath := range appCfg.Volumes { + // Host binds pass through untouched — see the single-server path. + if config.IsHostBindVolume(name) { + volumes[name] = containerPath + continue + } hostPath := fmt.Sprintf("/deployments/%s/volumes/%s", appCfg.App, name) volumes[hostPath] = containerPath - if _, err := s.exec.Run(ctx, fmt.Sprintf("mkdir -p %s", hostPath)); err != nil { + if _, err := s.exec.Run(ctx, "mkdir -p "+hostPath); err != nil { return fmt.Errorf("creating volume directory %s: %w", hostPath, err) } } diff --git a/internal/config/app.go b/internal/config/app.go index 72cad5b..fe8392d 100644 --- a/internal/config/app.go +++ b/internal/config/app.go @@ -801,6 +801,33 @@ func ValidateDomain(domain string, allowEmpty bool) error { return nil } +// IsHostBindVolume reports whether a volume key declares a HOST BIND: an +// absolute path the operator owns and teploy mounts exactly as given. Named +// volumes (the validName grammar) are teploy-managed under +// /deployments//volumes/; a bind is how a process receives +// something that already lives at a specific host path — a worker's trusted +// checkout and its deploy credentials, most famously — and teploy must not +// create, move, or back it up as if it were app data. +func IsHostBindVolume(name string) bool { + return strings.HasPrefix(name, "/") +} + +// validVolumeDestination checks the container side of a volume mapping: an +// absolute path, optionally carrying a docker mount-mode suffix (:ro/:rw). +// Relative destinations are refused — a typo'd "data" would silently create +// a path inside whatever cwd docker resolves, which is never what anyone +// wrote. +func validVolumeDestination(dest string) error { + mount := dest + if strings.HasSuffix(mount, ":ro") || strings.HasSuffix(mount, ":rw") { + mount = mount[:strings.LastIndex(mount, ":")] + } + if !strings.HasPrefix(mount, "/") || strings.ContainsAny(mount, "\r\n\x00") { + return fmt.Errorf("container destination must be an absolute path (a :ro/:rw mode suffix is allowed), got %q", dest) + } + return nil +} + func (c *AppConfig) validate() error { if err := ValidateName(c.App); err != nil { return err @@ -952,9 +979,12 @@ func (c *AppConfig) validate() error { if c.Health.IntervalSeconds < 0 { return fmt.Errorf("'health.interval_seconds' must be >= 0 (got %d)", c.Health.IntervalSeconds) } - for name := range c.Volumes { - if !validName.MatchString(name) { - return fmt.Errorf("volume name %q must be lowercase alphanumeric with hyphens", name) + for name, dest := range c.Volumes { + if !validName.MatchString(name) && !IsHostBindVolume(name) { + return fmt.Errorf("volume name %q must be lowercase alphanumeric with hyphens, or an absolute host path for a bind mount", name) + } + if err := validVolumeDestination(dest); err != nil { + return fmt.Errorf("volume %q: %w", name, err) } } for name, acc := range c.Accessories { diff --git a/internal/config/hardening_test.go b/internal/config/hardening_test.go index ae8c9a7..de008c1 100644 --- a/internal/config/hardening_test.go +++ b/internal/config/hardening_test.go @@ -134,6 +134,39 @@ volumes: } } +func TestLoadApp_HostBindVolumes(t *testing.T) { + dir := t.TempDir() + content := `app: myapp +domain: myapp.com +volumes: + "/srv/trusted-clone": "/srv/trusted-clone" + "/srv/creds": "/home/node/.ssh:ro" +` + os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte(content), 0644) + + cfg, err := LoadApp(dir) + if err != nil { + t.Fatalf("absolute-path volume keys are host binds and must load: %v", err) + } + if got := cfg.Volumes["/srv/creds"]; got != "/home/node/.ssh:ro" { + t.Fatalf("bind mount mode suffix must survive load, got %q", got) + } + if !IsHostBindVolume("/srv/trusted-clone") || IsHostBindVolume("app-data") { + t.Fatal("IsHostBindVolume must key on the leading slash, nothing else") + } + + // A relative destination is refused whichever side of the mapping failed. + bad := `app: myapp +domain: myapp.com +volumes: + "/srv/clone": "relative/path" +` + os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte(bad), 0644) + if _, err := LoadApp(dir); err == nil { + t.Fatal("expected error for a relative volume destination") + } +} + func TestLoadApp_TOML(t *testing.T) { dir := t.TempDir() content := `app = "myapp" From 0682f092cb35a2e0d48c4839f037b44bab9a4d7b Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:48:23 -0700 Subject: [PATCH 07/12] fix(preview,cli): canonical preview IDs end branch-slug collisions (C06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preview identity is now -p- (sha256 of app + full branch ref) through state files, container/process names, network aliases, Caddy route keys (stored in the record, not re-derived at teardown) and domains (slug becomes a display prefix with the hex suffix). Legacy slug-keyed records are adopted only when the stored full Branch matches (repo checked when both sides know it); a collision surfaces an explicit ambiguous-resource error naming both branches — never a guess, never a silent delete. Coexistence/update-one/destroy-one/prune-one/adoption/ ambiguity tested; golden-pinned ID derivation. The market-eval probe — red since the evaluation — is green. Lifecycle items (old-preview- serving-until-ready, expiry timer, preview profile, isolation) remain recorded C06 scope. --- AUDIT_OPEN.md | 88 +++++ internal/cli/preview.go | 58 ++++ internal/cli/preview_test.go | 39 +++ internal/preview/preview.go | 272 +++++++++++++-- internal/preview/preview_test.go | 546 ++++++++++++++++++++++++++++++- 5 files changed, 966 insertions(+), 37 deletions(-) create mode 100644 internal/cli/preview_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index d4b6484..43dae69 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -964,3 +964,91 @@ write — the EXISTING fence machinery refused both the guarded effect and the fenced state commit (ErrFenceLost). Result table preserved in the session receipt. The design spike's executable-proof obligation is met; the C01-1..C01-10 disagreement implementations remain open. + +## Programme slice (2026-09-22) — C06 preview canonical identity + +The product evaluation's branch-identity probe +(`_internal/evals/2026-09-21/probe_cli_contracts.py`, +TestMarketEvalPreviewBranchIdentityIsDistinct) demonstrated the C06 defect +live: `previewStatePath("market-eval", "feature/login")` == +`previewStatePath(..., "feature-login")` because SanitizeBranch strips both +`/` and `-` to the same slug — and the slug was the IDENTIFIER everywhere: +state file, container name/process, Caddy route key, and DNS label. Two +branches whose slugs collide silently shared (or fought over) one preview: +the second deploy destroyed the first's container and overwrote its record +and route. Base revision `22ae801`; changes left uncommitted for review. + +**Canonical ID design** (`internal/preview/preview.go`): + +- `PreviewID(app, branch)` = `-p-<8hex>`, 8hex = first 8 hex chars of + `sha256(app + NUL + full branch ref)`. The app IS the canonical repo + identity as teploy knows it (all server state is namespaced by it; one + app = one repo's deployment identity). The git remote URL is recorded + per-record as provenance but deliberately NOT hashed into the ID: remote + URLs change on repo renames and protocol switches, which would silently + orphan every existing preview. `previewIDHex` is pinned by + TestPreviewIDGolden so the scheme cannot drift unnoticed. +- Sanitized slugs are DISPLAY names only: they remain the human-readable + prefix of the preview subdomain, which now carries the ID suffix for + uniqueness — `preview--<8hex>.` (whole DNS label ≤ 63). + Two colliding branches therefore get distinct hostnames; without this, + coexistence would still break at the Caddy site block (one hostname, one + site). The slug never keys state, containers, or routes for new + resources; its remaining uses are the read-only legacy lookup and the + legacy-era route-key fallback. +- `State` records the full identity going forward: `id` (canonical), + `branch` (full, unsanitized — always was), `repo` (trivially normalized + origin remote: scheme/credentials stripped, `.git` dropped, scp-form + rewritten — `normalizeRepoURL`, internal/cli/preview.go), and `route` + (the Caddy route key / network alias this preview's artifacts live + under, making records self-describing instead of re-derived). + +**Identifier paths migrated** — state file `previewStatePath` +(preview.go:150), container name/process + network alias (Deploy, was +`-preview--`, now `-preview-p--`), Caddy +route key (SetRoute/RemoveRoute via `routeApp`/`previewRouteKey`), +preview domain (`previewDomain`), prune/cleanup enumeration (Prune/Destroy +resolve through the same keys), and the CLI create path records repo +identity (runPreviewDeploy → DeployConfig.Repo). Destroy-before-recreate +lifecycle behavior is unchanged this slice (separate recorded C06 item). + +**Legacy contract** (`resolveRecord`, preview.go): + +| Situation | Behavior | +|---|---| +| Legacy slug-keyed record, stored full Branch == requested (repo agrees when both record one) | ADOPT: Deploy migrates it under the canonical key with data preserved (full Branch, ID, Repo added), then tears down the artifacts the record itself names (stored Container, slug-era route key — `Route` empty marks the era); Destroy/Prune tear down those artifacts directly and remove the legacy file | +| Legacy record at the shared slug names a DIFFERENT branch (the collision), or repo mismatch when both record one | `*AmbiguousPreviewError` naming stored branch, requested branch, record path and remediation (`teploy preview destroy ` or manual rename/remove). NOTHING mutated — no container stop, no file removal, no route edit; verified by asserting the call log and file state stay empty/intact | +| Canonical record exists AND a mismatched legacy file sits at the shared slug | The legacy file belongs to the OTHER colliding branch: left untouched, does not block the operation (deploying/destroying this branch proceeds) | +| Canonical record exists AND a matching legacy duplicate exists | Interrupted-migration leftover of THIS branch (full-Branch match proves it): stale duplicate removed, canonical wins | +| Legacy record, unrelated slug | Keeps working untouched (TestPrune_OnlyDestroysExpired's fixtures are legacy records; TestLegacyOtherBranchNotBlocked covers cross-branch coexistence) | + +Destroy/Prune adopt on full-Branch match alone (they carry no repo +identity); repo participates wherever it is known (Deploy). A +present-but-unparseable record fails closed naming the path rather than +being treated as absent. + +**Evidence** — TDD: probe verified RED before the work +(TestMarketEvalPreviewBranchIdentityIsDistinct: +"feature/login" and "feature-login" → same record path), GREEN after; +probe suite fully green (compose contracts unchanged). New tests cover the +handoff's list, not just hash strings: coexistence (distinct state paths, +containers, routes, domains; neither deploy stops the other's container), +update-one-leaves-other, destroy-one-leaves-other, expire/prune-one +(modern + legacy fixtures), legacy adoption on Deploy (legacy file +migrated away, canonical record carries full Branch/ID/Repo/Route, legacy +container + slug route torn down), legacy collision ambiguity on Deploy +AND Destroy (record byte-identical after, zero mutation calls), repo +mismatch ambiguity, other-branch-not-blocked (including that branch's own +destroy still finding its legacy record), List over mixed-era records. +Mutation checks: constant ID suffix → identity/coexistence/golden tests +fail (colliding paths/domains/processes); removing the adoption +Branch-match → all three ambiguity tests fail (adopted instead of +refusing). Both reverted; gates after revert: `go vet ./...` clean, +`go test ./... -race` all packages ok, gofmt clean on touched files. + +**Remaining C06 scope (explicitly NOT in this slice)** — preview +lifecycle behavior (old preview serving until the new one is ready — +destroy-before-recreate stays as-is; expiry timer/automation beyond the +existing deploy-piggyback prune; config propagation through a preview +profile; network/secret isolation between previews) and any Dash-side +changes. diff --git a/internal/cli/preview.go b/internal/cli/preview.go index 43cc2a7..8a23482 100644 --- a/internal/cli/preview.go +++ b/internal/cli/preview.go @@ -5,7 +5,9 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "os/signal" + "strings" "time" "github.com/spf13/cobra" @@ -103,6 +105,10 @@ func runPreviewDeploy(flags *Flags, branch, ttlStr, image string) error { image = appCfg.App + "-build-" + version } + // Repo identity recorded in the preview record (provenance + legacy + // disambiguation). Not part of the preview ID — see gitRepoIdentity. + repo := gitRepoIdentity(".") + mgr := preview.NewManager(executor, os.Stdout) // Prune expired previews for this app before deploying a new one. @@ -128,6 +134,7 @@ func runPreviewDeploy(flags *Flags, branch, ttlStr, image string) error { Image: image, Version: version, TTL: ttl, + Repo: repo, }) if n := buildNotifier(appCfg); n != nil { @@ -277,3 +284,54 @@ func runPreviewPrune(flags *Flags) error { } return nil } + +// gitRepoIdentity returns the trivially normalized origin remote URL of the +// checkout at dir, or "" when it cannot be resolved (no git repo, no +// origin remote). The value is recorded in preview records as repo +// provenance and is compared when adopting legacy records; it is +// deliberately NOT hashed into the canonical preview ID — remote URLs +// change on repo renames and protocol switches, and keying identity on +// them would silently orphan every existing preview. +func gitRepoIdentity(dir string) string { + out, err := exec.Command("git", "-C", dir, "config", "--get", "remote.origin.url").Output() + if err != nil { + return "" + } + return normalizeRepoURL(string(out)) +} + +// normalizeRepoURL applies teploy's trivial repo-URL normalization: strip +// surrounding whitespace and a trailing ".git", strip the scheme and any +// user:token@ credentials, and rewrite the scp-like form to host/path — +// so https://git@github.com/o/r.git, git@github.com:o/r.git and +// ssh://git@github.com/o/r all record as github.com/o/r. This collapses +// the common spellings of one remote; anything else is recorded verbatim. +func normalizeRepoURL(raw string) string { + s := strings.TrimSuffix(strings.TrimSpace(raw), ".git") + if s == "" { + return "" + } + if i := strings.Index(s, "://"); i >= 0 { + s = s[i+3:] + if slash := strings.IndexByte(s, '/'); slash >= 0 { + authority, path := s[:slash], s[slash:] + if at := strings.LastIndexByte(authority, '@'); at >= 0 { + authority = authority[at+1:] + } + s = authority + path + } else if at := strings.LastIndexByte(s, '@'); at >= 0 { + s = s[at+1:] + } + return s + } + // scp-like form: [user@]host:path — the first colon before any slash + // separates host from path. + if i := strings.IndexByte(s, ':'); i > 0 && !strings.Contains(s[:i], "/") { + host := s[:i] + if at := strings.LastIndexByte(host, '@'); at >= 0 { + host = host[at+1:] + } + return host + "/" + s[i+1:] + } + return s +} diff --git a/internal/cli/preview_test.go b/internal/cli/preview_test.go new file mode 100644 index 0000000..8bc6810 --- /dev/null +++ b/internal/cli/preview_test.go @@ -0,0 +1,39 @@ +package cli + +import "testing" + +// Repo identity is provenance in preview records and a legacy-adoption +// check, never part of the preview ID (see gitRepoIdentity). The +// normalization is deliberately trivial: collapse the common spellings of +// one remote; record anything else verbatim. +func TestNormalizeRepoURL(t *testing.T) { + tests := []struct{ raw, want string }{ + {"https://github.com/useteploy/teploy-cli.git", "github.com/useteploy/teploy-cli"}, + {"https://github.com/useteploy/teploy-cli", "github.com/useteploy/teploy-cli"}, + {"git@github.com:useteploy/teploy-cli.git", "github.com/useteploy/teploy-cli"}, + {"ssh://git@github.com/useteploy/teploy-cli.git", "github.com/useteploy/teploy-cli"}, + {"https://user:token@github.com/o/r.git", "github.com/o/r"}, + {"git@gitlab.com:o/r.git", "gitlab.com/o/r"}, + {"git@100.108.123.49:tyler/teploy.git", "100.108.123.49/tyler/teploy"}, + {"", ""}, + {" ", ""}, + {"/local/path/repo.git", "/local/path/repo"}, + } + for _, tt := range tests { + if got := normalizeRepoURL(tt.raw); got != tt.want { + t.Errorf("normalizeRepoURL(%q) = %q, want %q", tt.raw, got, tt.want) + } + } +} + +// gitRepoIdentity resolves from a real checkout and returns "" when no +// origin remote exists. +func TestGitRepoIdentity(t *testing.T) { + dir := t.TempDir() + if got := gitRepoIdentity(dir); got != "" { + t.Errorf("empty dir must yield empty identity, got %q", got) + } + if got := gitRepoIdentity(t.TempDir() + "/nonexistent"); got != "" { + t.Errorf("missing dir must yield empty identity, got %q", got) + } +} diff --git a/internal/preview/preview.go b/internal/preview/preview.go index 8ebbb37..3efed3f 100644 --- a/internal/preview/preview.go +++ b/internal/preview/preview.go @@ -2,6 +2,8 @@ package preview import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" @@ -18,7 +20,22 @@ const deploymentsDir = "/deployments" // State tracks a preview deployment on the server. type State struct { - Branch string `json:"branch"` + // ID is the canonical preview identifier (-p-<8hex>, see + // PreviewID). Empty on records written before the canonical-ID + // migration (legacy slug-keyed records). + ID string `json:"id,omitempty"` + // Branch is the FULL, unsanitized branch name. Records are keyed by + // the canonical ID, not by this value's sanitized form. + Branch string `json:"branch"` + // Repo is the trivially normalized origin remote URL of the checkout + // that deployed this preview ("" when unresolvable). Provenance and + // legacy disambiguation only — deliberately not part of the ID, which + // keys on the app (the repo's stable deployment identity) instead. + Repo string `json:"repo,omitempty"` + // Route is the Caddy route key / docker network alias this preview's + // artifacts live under. Empty on legacy records, whose artifacts were + // keyed by the sanitized branch slug. + Route string `json:"route,omitempty"` Domain string `json:"domain"` Port int `json:"port"` Container string `json:"container"` @@ -38,6 +55,10 @@ type DeployConfig struct { Env map[string]string Volumes map[string]string TTL time.Duration // default 72h + // Repo is the normalized repo identity recorded in the preview record + // (see State.Repo). Empty is allowed: the repo is provenance, not part + // of the preview ID. + Repo string } // Manager handles preview environment lifecycle. @@ -60,7 +81,10 @@ func NewManager(exec ssh.Executor, out io.Writer) *Manager { var nonAlphanumeric = regexp.MustCompile(`[^a-z0-9-]`) -// SanitizeBranch cleans a branch name for use in DNS labels. +// SanitizeBranch cleans a branch name for use in DNS labels. This is a +// DISPLAY derivation only: distinct branches can sanitize to the same slug +// (feature/login and feature-login both become feature-login), so it must +// never key state, containers, or routes — that is PreviewID's job. func SanitizeBranch(branch string) string { s := strings.ToLower(branch) s = strings.ReplaceAll(s, "/", "-") @@ -79,28 +103,220 @@ func SanitizeBranch(branch string) string { return s } -// previewDomain returns the subdomain for a preview: preview-{branch}.{domain} -func previewDomain(branch, baseDomain string) string { - return fmt.Sprintf("preview-%s.%s", SanitizeBranch(branch), baseDomain) +// previewIDHex returns the collision-resistant identity suffix for a +// preview: the first 8 hex chars of sha256(app + NUL + full branch ref). +// The app is the canonical repo identity as teploy knows it — every piece +// of server state is namespaced by the app, so one app is one repo's +// deployment identity. The git remote URL is recorded per-record as +// provenance but deliberately NOT hashed into the ID: remote URLs change +// on repo renames and protocol switches, which would silently orphan +// existing previews, while the app name is stable. +func previewIDHex(app, branch string) string { + sum := sha256.Sum256([]byte(app + "\x00" + branch)) + return hex.EncodeToString(sum[:4]) +} + +// PreviewID returns the canonical preview identifier: -p-<8hex>, +// derived from the app (canonical repo identity) plus the full branch ref. +// Distinct branches — even ones whose sanitized slugs collide — always get +// distinct IDs, and therefore distinct state files, containers, routes, +// and domains. +func PreviewID(app, branch string) string { + return app + "-p-" + previewIDHex(app, branch) +} + +// previewDomain returns the per-preview subdomain. The sanitized branch is +// the human-readable display part; the canonical ID suffix guarantees +// uniqueness, so branches that sanitize identically (feature/login vs +// feature-login) get distinct hostnames instead of fighting over one +// Caddy site block. The slug is bounded so the whole DNS label stays +// within 63 characters: "preview-" (8) + slug + "-" + 8hex. +func previewDomain(app, branch, baseDomain string) string { + slug := SanitizeBranch(branch) + if max := 63 - len("preview-") - 1 - len(previewIDHex(app, branch)); len(slug) > max { + slug = strings.TrimRight(slug[:max], "-") + } + return fmt.Sprintf("preview-%s-%s.%s", slug, previewIDHex(app, branch), baseDomain) } func previewDir(app string) string { return fmt.Sprintf("%s/%s/previews", deploymentsDir, app) } +// previewStatePath is the canonical state-file path, keyed by PreviewID. func previewStatePath(app, branch string) string { + return fmt.Sprintf("%s/%s.json", previewDir(app), PreviewID(app, branch)) +} + +// legacyPreviewStatePath is the pre-canonical-ID state-file path, keyed by +// the sanitized branch slug. Read-only: used to find and adopt (or refuse) +// records written by older teploy versions. Never written. +func legacyPreviewStatePath(app, branch string) string { return fmt.Sprintf("%s/%s.json", previewDir(app), SanitizeBranch(branch)) } +// previewRouteKey derives the Caddy route key (and docker network alias) +// under which a record's artifacts live. Modern records carry the key in +// State.Route; records without one predate the field and their artifacts +// were keyed by the sanitized branch slug. +func previewRouteKey(app string, s *State) string { + if s.Route != "" { + return s.Route + } + return app + "-preview-" + SanitizeBranch(s.Branch) +} + +// AmbiguousPreviewError reports a legacy slug-keyed preview record whose +// identity cannot be established for the requested branch: the file is +// keyed by the sanitized slug two or more branches share, and its stored +// full Branch (or repo, when both sides record one) does not match the +// request. The record is NEVER mutated or deleted in this case — the +// operator must disambiguate explicitly. +type AmbiguousPreviewError struct { + App string + Path string + StoredBranch string + RequestedBranch string + StoredRepo string + RequestedRepo string +} + +func (e *AmbiguousPreviewError) Error() string { + detail := fmt.Sprintf("stored branch %q does not match requested branch %q", e.StoredBranch, e.RequestedBranch) + if e.StoredRepo != "" || e.RequestedRepo != "" { + detail += fmt.Sprintf(" (stored repo %q vs requested repo %q)", e.StoredRepo, e.RequestedRepo) + } + return fmt.Sprintf( + "ambiguous legacy preview record for app %q at %s: %s — the sanitized slug is shared by multiple branches, so identity cannot be established automatically. "+ + "Destroy the recorded preview explicitly with its own branch (`teploy preview destroy %s`), or inspect and remove/rename the state file on the server. Nothing was changed.", + e.App, e.Path, detail, e.StoredBranch, + ) +} + +// readRecord reads and parses the preview record at path. A confirmed +// absent file returns (nil, nil); a present-but-unparseable record is an +// error naming the path (identity cannot be established — fail closed +// rather than guessing or silently adopting). +func (m *Manager) readRecord(ctx context.Context, path string) (*State, error) { + content, err := m.exec.Run(ctx, "cat "+path) + if err != nil || strings.TrimSpace(content) == "" { + return nil, nil + } + var s State + if err := json.Unmarshal([]byte(strings.TrimSpace(content)), &s); err != nil { + return nil, fmt.Errorf("reading preview record %s: %w", path, err) + } + return &s, nil +} + +// resolveRecord locates the preview record for (app, branch) across the +// canonical-ID key and the legacy slug key, and returns it together with +// the path it was read from (nil, "" when no record exists). +// +// Legacy contract: a slug-keyed record is only touched when its stored +// full Branch matches the requested branch exactly (and its recorded Repo +// agrees when both sides have one). A match found at the legacy key is +// returned as-is with its path — callers decide what adoption means for +// their operation; Deploy migrates it to the canonical key, Destroy tears +// down the artifacts it actually names. A mismatch is an +// *AmbiguousPreviewError and nothing is mutated. When both keys hold +// records, the canonical one wins; a legacy duplicate whose stored Branch +// matches this branch is a stale leftover of an interrupted migration and +// is removed, but a legacy record for a DIFFERENT colliding branch belongs +// to that branch and is left in place untouched. +func (m *Manager) resolveRecord(ctx context.Context, app, branch, repo string) (*State, string, error) { + canonPath := previewStatePath(app, branch) + canon, err := m.readRecord(ctx, canonPath) + if err != nil { + return nil, "", err + } + + legacyPath := legacyPreviewStatePath(app, branch) + legacy, err := m.readRecord(ctx, legacyPath) + if err != nil { + return nil, "", err + } + if legacy == nil { + return canon, canonPath, nil + } + + // Only when the canonical key holds nothing can the legacy record + // become this branch's: then its identity must be established exactly. + // When the canonical key already holds this branch's record, a legacy + // file under the shared slug that names a DIFFERENT branch belongs to + // that branch's own (legacy) preview and must not block or color this + // operation at all. + if canon == nil { + if legacy.Branch != branch || (legacy.Repo != "" && repo != "" && legacy.Repo != repo) { + return nil, "", &AmbiguousPreviewError{ + App: app, + Path: legacyPath, + StoredBranch: legacy.Branch, + RequestedBranch: branch, + StoredRepo: legacy.Repo, + RequestedRepo: repo, + } + } + return legacy, legacyPath, nil + } + // Both keys hold records. A legacy file whose stored Branch matches + // this branch is a stale duplicate of the canonical one (interrupted + // migration) — the full-Branch match established it is this branch's + // own, so remove it. Any other legacy file stays untouched above. + if legacy.Branch == branch { + m.exec.Run(ctx, "rm -f -- "+legacyPath) + } + return canon, canonPath, nil +} + +// writeRecord persists a preview record at path. +func (m *Manager) writeRecord(ctx context.Context, s *State, path string) error { + data, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + if err := m.exec.Upload(ctx, strings.NewReader(string(data)), path, "0644"); err != nil { + return err + } + return nil +} + // Deploy creates or updates a preview environment for the given branch. func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { if cfg.TTL == 0 { cfg.TTL = 72 * time.Hour } - sanitized := SanitizeBranch(cfg.Branch) - domain := previewDomain(cfg.Branch, cfg.Domain) - containerName := fmt.Sprintf("%s-preview-%s-%s", cfg.App, sanitized, cfg.Version) + // Resolve any legacy record BEFORE mutating anything: a slug-keyed + // record that belongs to a different branch must stop the deploy with + // an ambiguous-resource error, never be silently overwritten; one that + // unambiguously belongs to this branch is adopted under the canonical + // key first, so the rewrite below replaces one record instead of + // orphaning the old key. + existing, existingPath, err := m.resolveRecord(ctx, cfg.App, cfg.Branch, cfg.Repo) + if err != nil { + return err + } + if existing != nil && existingPath == legacyPreviewStatePath(cfg.App, cfg.Branch) { + adopted := *existing + adopted.ID = PreviewID(cfg.App, cfg.Branch) + if adopted.Repo == "" { + adopted.Repo = cfg.Repo + } + // The record's live artifacts predate the migration (Route empty, + // slug-keyed) — keep them described exactly as they are so the + // destroy below tears down what is actually running. + if err := m.writeRecord(ctx, &adopted, previewStatePath(cfg.App, cfg.Branch)); err != nil { + return fmt.Errorf("migrating legacy preview record: %w", err) + } + m.exec.Run(ctx, "rm -f -- "+existingPath) + } + + idHex := previewIDHex(cfg.App, cfg.Branch) + domain := previewDomain(cfg.App, cfg.Branch, cfg.Domain) + process := "preview-p-" + idHex + routeApp := cfg.App + "-" + process + containerName := fmt.Sprintf("%s-%s-%s", cfg.App, process, cfg.Version) fmt.Fprintf(m.out, "Deploying preview for branch %q...\n", cfg.Branch) fmt.Fprintf(m.out, " Domain: %s\n", domain) @@ -127,7 +343,7 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { } _, err = m.docker.Run(ctx, docker.RunConfig{ App: cfg.App, - Process: "preview-" + sanitized, + Process: process, Version: cfg.Version, Image: cfg.Image, Port: port, @@ -139,10 +355,9 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { return fmt.Errorf("starting preview container: %w", err) } - // Set Caddy route for preview domain. The preview container gets a - // dedicated network alias (cfg.App + "-preview-" + sanitized) via + // Set Caddy route for the preview domain. The preview container gets a + // dedicated network alias (cfg.App + "-" + process) via // docker.RunConfig.Process, which is what we dial here. - routeApp := cfg.App + "-preview-" + sanitized // Caddy dials the upstream over the docker network, so it needs the // container's INTERNAL port, not the host-published port (which is what // `port` is). Passing the host port made Caddy dial a port the container @@ -164,7 +379,10 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { // Write state. now := time.Now().UTC() state := State{ + ID: PreviewID(cfg.App, cfg.Branch), Branch: cfg.Branch, + Repo: cfg.Repo, + Route: routeApp, Domain: domain, Port: port, Container: containerName, @@ -172,10 +390,7 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { CreatedAt: now, ExpiresAt: now.Add(cfg.TTL), } - - data, _ := json.MarshalIndent(state, "", " ") - statePath := previewStatePath(cfg.App, cfg.Branch) - if err := m.exec.Upload(ctx, strings.NewReader(string(data)), statePath, "0644"); err != nil { + if err := m.writeRecord(ctx, &state, previewStatePath(cfg.App, cfg.Branch)); err != nil { return fmt.Errorf("writing preview state: %w", err) } @@ -184,7 +399,9 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { return nil } -// List returns all active previews for the app. +// List returns all active previews for the app. Records from both the +// canonical-ID keys and legacy slug keys are listed; legacy records are +// returned unmodified (readers never mutate). func (m *Manager) List(ctx context.Context, app string) ([]State, error) { dir := previewDir(app) out, err := m.exec.Run(ctx, fmt.Sprintf("ls %s/*.json 2>/dev/null", dir)) @@ -213,28 +430,23 @@ func (m *Manager) List(ctx context.Context, app string) ([]State, error) { // Destroy tears down a preview environment. func (m *Manager) Destroy(ctx context.Context, app, branch string) error { - statePath := previewStatePath(app, branch) - content, err := m.exec.Run(ctx, "cat "+statePath+" 2>/dev/null") - if err != nil || strings.TrimSpace(content) == "" { - return nil // no preview to destroy + s, path, err := m.resolveRecord(ctx, app, branch, "") + if err != nil { + return err } - - var s State - if err := json.Unmarshal([]byte(strings.TrimSpace(content)), &s); err != nil { - return nil + if s == nil { + return nil // no preview to destroy } // Stop and remove container. m.docker.Stop(ctx, s.Container, 5) m.docker.Remove(ctx, s.Container) - // Remove Caddy route. - sanitized := SanitizeBranch(branch) - routeApp := app + "-preview-" + sanitized - m.caddy.RemoveRoute(ctx, routeApp) + // Remove Caddy route (keyed by the record's own era). + m.caddy.RemoveRoute(ctx, previewRouteKey(app, s)) // Remove state file. - m.exec.Run(ctx, "rm -f "+statePath) + m.exec.Run(ctx, "rm -f -- "+path) fmt.Fprintf(m.out, "Destroyed preview for branch %q\n", branch) return nil diff --git a/internal/preview/preview_test.go b/internal/preview/preview_test.go index 6987247..ec4251d 100644 --- a/internal/preview/preview_test.go +++ b/internal/preview/preview_test.go @@ -3,9 +3,12 @@ package preview import ( "bytes" "context" + "encoding/json" + "errors" "fmt" "strings" "testing" + "time" "github.com/useteploy/teploy/internal/ssh" ) @@ -32,6 +35,61 @@ func TestSanitizeBranch(t *testing.T) { } } +// The canonical-ID derivation is pinned to exact sha256 output so a silent +// change to the identity scheme (which would orphan every deployed preview +// record) cannot land unnoticed. Values are sha256("\x00") +// truncated to 8 hex chars. +func TestPreviewIDGolden(t *testing.T) { + tests := []struct { + app, branch, want string + }{ + {"myapp", "feature/login", "myapp-p-08e81639"}, + {"myapp", "feature-login", "myapp-p-cb4bdf9a"}, + {"myapp", "main", "myapp-p-563059ce"}, + {"myapp", "old-feature", "myapp-p-491218d3"}, + {"myapp", "active-feature", "myapp-p-9fdbab8f"}, + {"market-eval", "feature/login", "market-eval-p-a575aaf7"}, + {"market-eval", "feature-login", "market-eval-p-84d280a4"}, + } + for _, tt := range tests { + if got := PreviewID(tt.app, tt.branch); got != tt.want { + t.Errorf("PreviewID(%q, %q) = %q, want %q", tt.app, tt.branch, got, tt.want) + } + if got := previewIDHex(tt.app, tt.branch); got != strings.TrimPrefix(tt.want, tt.app+"-p-") { + t.Errorf("previewIDHex(%q, %q) = %q, want %q", tt.app, tt.branch, got, strings.TrimPrefix(tt.want, tt.app+"-p-")) + } + // Deterministic: the same identity inputs always derive the same ID. + if PreviewID(tt.app, tt.branch) != PreviewID(tt.app, tt.branch) { + t.Errorf("PreviewID(%q, %q) not deterministic", tt.app, tt.branch) + } + } +} + +// The C06 defect in one test: feature/login and feature/login's slug twin +// feature-login must never share an identifier anywhere — state path, +// container/process name, route key, or DNS label. +func TestPreviewBranchIdentityIsDistinct(t *testing.T) { + left, right := "feature/login", "feature-login" + if previewStatePath("myapp", left) == previewStatePath("myapp", right) { + t.Errorf("state paths collide: %q", previewStatePath("myapp", left)) + } + if legacyPreviewStatePath("myapp", left) != legacyPreviewStatePath("myapp", right) { + t.Errorf("legacy slug paths must collide (that is the defect being migrated): %q vs %q", + legacyPreviewStatePath("myapp", left), legacyPreviewStatePath("myapp", right)) + } + if previewDomain("myapp", left, "myapp.com") == previewDomain("myapp", right, "myapp.com") { + t.Errorf("domains collide: %q", previewDomain("myapp", left, "myapp.com")) + } + leftProc := "preview-p-" + previewIDHex("myapp", left) + rightProc := "preview-p-" + previewIDHex("myapp", right) + if leftProc == rightProc { + t.Errorf("container process names collide: %q", leftProc) + } + if "myapp-"+leftProc == "myapp-"+rightProc { + t.Errorf("route keys collide") + } +} + func TestDeploy(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", ssh.MockCommand{Match: "mkdir -p /deployments/myapp/previews", Output: ""}, @@ -66,7 +124,7 @@ func TestDeploy(t *testing.T) { } output := buf.String() - if !bytes.Contains([]byte(output), []byte("preview-feature-login.myapp.com")) { + if !bytes.Contains([]byte(output), []byte("preview-feature-login-08e81639.myapp.com")) { t.Errorf("expected preview domain in output, got: %s", output) } } @@ -94,6 +152,9 @@ func TestList_Empty(t *testing.T) { // `teploy preview deploy` so expired previews for the app get torn down // (container + Caddy route) before a new one is created. This test proves // Prune destroys an expired preview and leaves a non-expired one alone. +// The fixtures are legacy slug-keyed records, so it doubles as the +// prune-side legacy-adoption proof: Destroy adopts the expired legacy +// record by full-Branch match and tears down exactly its artifacts. func TestPrune_OnlyDestroysExpired(t *testing.T) { expiredJSON := `{"branch":"old-feature","domain":"preview-old-feature.myapp.com","port":49200,"container":"myapp-preview-old-feature-v1","image":"myapp:v1","created_at":"2020-01-01T00:00:00Z","expires_at":"2020-01-02T00:00:00Z"}` freshJSON := fmt.Sprintf(`{"branch":"active-feature","domain":"preview-active-feature.myapp.com","port":49201,"container":"myapp-preview-active-feature-v2","image":"myapp:v2","created_at":"2020-01-01T00:00:00Z","expires_at":%q}`, @@ -116,7 +177,7 @@ func TestPrune_OnlyDestroysExpired(t *testing.T) { ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, - ssh.MockCommand{Match: "rm -f /deployments/myapp/previews/old-feature.json", Output: ""}, + ssh.MockCommand{Match: "rm -f -- /deployments/myapp/previews/old-feature.json", Output: ""}, ) var buf bytes.Buffer @@ -139,15 +200,486 @@ func TestPrune_OnlyDestroysExpired(t *testing.T) { func TestPreviewDomain(t *testing.T) { tests := []struct { - branch, domain, want string + app, branch, domain, want string }{ - {"feature/login", "myapp.com", "preview-feature-login.myapp.com"}, - {"main", "example.com", "preview-main.example.com"}, + {"myapp", "feature/login", "myapp.com", "preview-feature-login-08e81639.myapp.com"}, + {"myapp", "main", "example.com", "preview-main-563059ce.example.com"}, + // A 70-char slug must truncate so the whole DNS label stays <= 63: + // "preview-" (8) + 46 chars + "-" + 8 hex = 63. + {"myapp", strings.Repeat("a", 70) + "/x", "myapp.com", "preview-" + strings.Repeat("a", 46) + "-54119e3a.myapp.com"}, } for _, tt := range tests { - got := previewDomain(tt.branch, tt.domain) + got := previewDomain(tt.app, tt.branch, tt.domain) if got != tt.want { - t.Errorf("previewDomain(%q, %q) = %q, want %q", tt.branch, tt.domain, got, tt.want) + t.Errorf("previewDomain(%q, %q, %q) = %q, want %q", tt.app, tt.branch, tt.domain, got, tt.want) + } + label := strings.SplitN(got, ".", 2)[0] + if len(label) > 63 { + t.Errorf("DNS label %q exceeds 63 chars (%d)", label, len(label)) + } + } +} + +// previewDeployMocks is the mock bundle for a full Deploy against a bare +// server: port allocation, container start, and the Caddyfile +// edit/reload/verify transaction (see TestDeploy for the origins of each +// entry). State-file and Caddyfile writes go through the mock's file +// state, so successive deploys observe each other's records and routes. +func previewDeployMocks() []ssh.MockCommand { + return []ssh.MockCommand{ + ssh.MockCommand{Match: "mkdir -p /deployments/myapp/previews", Output: ""}, + ssh.MockCommand{Match: "ss -tln", Output: ""}, + ssh.MockCommand{Match: "docker run", Output: "abc123"}, + ssh.MockCommand{Match: "docker inspect -f '{{range $p", Output: "80/tcp"}, + ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + } +} + +const ( + loginIDHex = "08e81639" // previewIDHex("myapp", "feature/login") + dashIDHex = "cb4bdf9a" // previewIDHex("myapp", "feature-login") + loginBranch = "feature/login" + dashBranch = "feature-login" +) + +func deployCfg(branch, version string) DeployConfig { + return DeployConfig{ + App: "myapp", + Domain: "myapp.com", + Branch: branch, + Image: "myapp:" + version, + Version: version, + Repo: "github.com/tyler/myapp", + } +} + +func mustDeploy(t *testing.T, mgr *Manager, cfg DeployConfig) { + t.Helper() + if err := mgr.Deploy(context.Background(), cfg); err != nil { + t.Fatalf("Deploy(%q): %v", cfg.Branch, err) + } +} + +func callsContaining(mock *ssh.MockExecutor, needle string) []string { + var found []string + for _, c := range mock.Calls { + if strings.Contains(c, needle) { + found = append(found, c) + } + } + return found +} + +// The C06 coexistence contract: two branches whose sanitized slugs collide +// deploy side by side with distinct state records, containers, routes and +// domains — neither deploy destroys or blocks the other. +func TestDeployCoexistence(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + mustDeploy(t, mgr, deployCfg(dashBranch, "v1")) + + loginPath := previewStatePath("myapp", loginBranch) + dashPath := previewStatePath("myapp", dashBranch) + if loginPath == dashPath { + t.Fatalf("state paths collide: %q", loginPath) + } + for _, path := range []string{loginPath, dashPath} { + if _, ok := mock.Files[path]; !ok { + t.Errorf("expected state record at %s", path) + } + } + + // Distinct containers were started; neither deploy tore anything down + // (there was nothing to destroy), proving the second deploy did not + // resolve to the first preview's identity. + if got := callsContaining(mock, "docker run"); len(got) != 2 { + t.Fatalf("expected 2 container starts, got %d: %v", len(got), got) + } + if len(callsContaining(mock, "--name 'myapp-preview-p-"+loginIDHex+"-v1'")) != 1 { + t.Errorf("missing container name myapp-preview-p-%s-v1 in: %v", loginIDHex, callsContaining(mock, "docker run")) + } + if len(callsContaining(mock, "--name 'myapp-preview-p-"+dashIDHex+"-v1'")) != 1 { + t.Errorf("missing container name myapp-preview-p-%s-v1 in: %v", dashIDHex, callsContaining(mock, "docker run")) + } + if stops := callsContaining(mock, "docker stop"); len(stops) != 0 { + t.Errorf("fresh deploys must not stop containers, got: %v", stops) + } + + // Both records carry their full branch identity and distinct routes. + var loginState, dashState State + if err := json.Unmarshal(mock.Files[loginPath], &loginState); err != nil { + t.Fatalf("login record: %v", err) + } + if err := json.Unmarshal(mock.Files[dashPath], &dashState); err != nil { + t.Fatalf("dash record: %v", err) + } + if loginState.Branch != loginBranch || dashState.Branch != dashBranch { + t.Errorf("full branch identity not preserved: %+v / %+v", loginState, dashState) + } + if loginState.Route == dashState.Route { + t.Errorf("route keys collide: %q", loginState.Route) + } + + // Both routes and domains are distinct and live. + caddy := string(mock.Files["/deployments/caddy/Caddyfile"]) + for _, key := range []string{"myapp-preview-p-" + loginIDHex, "myapp-preview-p-" + dashIDHex} { + if !strings.Contains(caddy, key) { + t.Errorf("Caddyfile missing route key %s", key) + } + } + out := buf.String() + for _, domain := range []string{ + "preview-feature-login-" + loginIDHex + ".myapp.com", + "preview-feature-login-" + dashIDHex + ".myapp.com", + } { + if !strings.Contains(out, domain) { + t.Errorf("output missing distinct domain %s", domain) + } + } +} + +// Updating one preview of a colliding pair must not touch the other's +// record or container. +func TestDeployUpdateOneLeavesOther(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + mustDeploy(t, mgr, deployCfg(dashBranch, "v1")) + dashPath := previewStatePath("myapp", dashBranch) + dashBefore := string(mock.Files[dashPath]) + + mustDeploy(t, mgr, deployCfg(loginBranch, "v2")) + + // The redeploy replaced only its own generation. + if len(callsContaining(mock, "docker stop -t 5 'myapp-preview-p-"+loginIDHex+"-v1'")) != 1 { + t.Errorf("expected the login v1 container to be stopped, calls: %v", callsContaining(mock, "docker stop")) + } + if stops := callsContaining(mock, "docker stop"); len(stops) != 1 { + t.Errorf("redeploy must stop exactly one container, got: %v", stops) + } + if got := string(mock.Files[dashPath]); got != dashBefore { + t.Errorf("the colliding branch's record was modified:\nbefore: %s\nafter: %s", dashBefore, got) + } +} + +// Destroying one preview of a colliding pair leaves the other fully +// intact. +func TestDestroyOneLeavesOther(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + mustDeploy(t, mgr, deployCfg(dashBranch, "v1")) + dashPath := previewStatePath("myapp", dashBranch) + dashBefore := string(mock.Files[dashPath]) + + if err := mgr.Destroy(context.Background(), "myapp", loginBranch); err != nil { + t.Fatalf("Destroy: %v", err) + } + + if _, ok := mock.Files[previewStatePath("myapp", loginBranch)]; ok { + t.Errorf("destroyed preview's record still present") + } + if got := string(mock.Files[dashPath]); got != dashBefore { + t.Errorf("the colliding branch's record was modified:\nbefore: %s\nafter: %s", dashBefore, got) + } + if len(callsContaining(mock, "docker stop -t 5 'myapp-preview-p-"+dashIDHex+"-v1'")) != 0 { + t.Errorf("the colliding branch's container must not be stopped, calls: %v", callsContaining(mock, "docker stop")) + } + // Its route survives too: only the login route key was removed. + caddy := string(mock.Files["/deployments/caddy/Caddyfile"]) + if strings.Contains(caddy, "myapp-preview-p-"+loginIDHex) { + t.Errorf("destroyed preview's route still present in Caddyfile") + } + if !strings.Contains(caddy, "myapp-preview-p-"+dashIDHex) { + t.Errorf("surviving preview's route missing from Caddyfile") + } +} + +// Prune expires one canonical preview of a colliding pair and leaves the +// other running. +func TestPruneExpiresOneLeavesOther(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + append([]ssh.MockCommand{ + ssh.MockCommand{Match: "ls /deployments/myapp/previews/*.json", + Output: previewStatePath("myapp", loginBranch) + "\n" + previewStatePath("myapp", dashBranch)}, + }, previewDeployMocks()...)...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + mustDeploy(t, mgr, deployCfg(dashBranch, "v1")) + dashPath := previewStatePath("myapp", dashBranch) + dashBefore := string(mock.Files[dashPath]) + + // Expire the login preview by rewriting its stored record. + loginPath := previewStatePath("myapp", loginBranch) + var s State + if err := json.Unmarshal(mock.Files[loginPath], &s); err != nil { + t.Fatalf("login record: %v", err) + } + s.ExpiresAt = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + expired, _ := json.Marshal(s) + mock.Files[loginPath] = expired + + pruned, err := mgr.Prune(context.Background(), "myapp") + if err != nil { + t.Fatalf("Prune: %v", err) + } + if pruned != 1 { + t.Fatalf("expected 1 pruned preview, got %d", pruned) + } + if _, ok := mock.Files[loginPath]; ok { + t.Errorf("expired preview's record still present") + } + if got := string(mock.Files[dashPath]); got != dashBefore { + t.Errorf("the colliding branch's record was modified:\nbefore: %s\nafter: %s", dashBefore, got) + } + if len(callsContaining(mock, "docker stop -t 5 'myapp-preview-p-"+dashIDHex+"-v1'")) != 0 { + t.Errorf("the colliding branch's container must not be stopped, calls: %v", callsContaining(mock, "docker stop")) + } +} + +// legacyRecordJSON is a record exactly as the pre-canonical-ID writer +// emitted it: slug-keyed file, full Branch, no ID/Repo/Route fields. +func legacyRecordJSON(branch, container string) string { + return fmt.Sprintf(`{"branch":%q,"domain":"preview-%s.myapp.com","port":49200,"container":%q,"image":"myapp:v1","created_at":"2020-01-01T00:00:00Z","expires_at":"2099-01-01T00:00:00Z"}`, + branch, SanitizeBranch(branch), container) +} + +// seedCaddyfileWithRoute seeds the Caddyfile file state with a managed +// block for an app key, mimicking what an earlier teploy version left on +// the server. +func seedCaddyfileWithRoute(mock *ssh.MockExecutor, key, host string) { + mock.Files["/deployments/caddy/Caddyfile"] = []byte(fmt.Sprintf( + "{\n\tadmin 0.0.0.0:2019\n}\n\n# TEPLOY BEGIN %s\n%s {\n\treverse_proxy %s:80\n}\n# TEPLOY END %s\n", + key, host, key, key)) +} + +// A legacy slug-keyed record whose stored full Branch matches is adopted: +// Deploy migrates it under the canonical key (identity preserved), tears +// down the artifacts the record actually names (the old slug-keyed +// container and route), and deploys fresh canonical-keyed artifacts. +func TestLegacyAdoptionDeploy(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + legacyPath := legacyPreviewStatePath("myapp", loginBranch) // .../feature-login.json + mock.Files[legacyPath] = []byte(legacyRecordJSON(loginBranch, "myapp-preview-feature-login-v1")) + seedCaddyfileWithRoute(mock, "myapp-preview-feature-login", "preview-feature-login.myapp.com") + + mustDeploy(t, mgr, deployCfg(loginBranch, "v2")) + + // The legacy file is gone; the canonical record carries the FULL + // branch identity, the canonical ID, the repo provenance, and the new + // route key. + if _, ok := mock.Files[legacyPath]; ok { + t.Errorf("legacy record was not migrated away from %s", legacyPath) + } + canonPath := previewStatePath("myapp", loginBranch) + data, ok := mock.Files[canonPath] + if !ok { + t.Fatalf("canonical record missing at %s", canonPath) + } + var s State + if err := json.Unmarshal(data, &s); err != nil { + t.Fatalf("canonical record: %v", err) + } + if s.Branch != loginBranch { + t.Errorf("full branch identity lost in adoption: %+v", s) + } + if s.ID != "myapp-p-"+loginIDHex { + t.Errorf("canonical ID missing/wrong: %+v", s) + } + if s.Repo != "github.com/tyler/myapp" { + t.Errorf("repo provenance not recorded: %+v", s) + } + if s.Route != "myapp-preview-p-"+loginIDHex { + t.Errorf("route key not recorded: %+v", s) + } + + // The legacy record's OWN artifacts were torn down: its container and + // its slug-era route. + if len(callsContaining(mock, "docker stop -t 5 'myapp-preview-feature-login-v1'")) != 1 { + t.Errorf("legacy container not stopped via its stored name, calls: %v", callsContaining(mock, "docker stop")) + } + caddy := string(mock.Files["/deployments/caddy/Caddyfile"]) + if strings.Contains(caddy, "myapp-preview-feature-login") { + t.Errorf("legacy slug-keyed route not removed:\n%s", caddy) + } + if !strings.Contains(caddy, "myapp-preview-p-"+loginIDHex) { + t.Errorf("new canonical route missing:\n%s", caddy) + } +} + +// The collision case: a legacy record at the shared slug path belongs to a +// DIFFERENT branch. Deploy must refuse with an ambiguous-resource error +// naming both branches and the record path, and must not mutate anything. +func TestLegacyCollisionDeployIsAmbiguous(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + legacyPath := legacyPreviewStatePath("myapp", loginBranch) // shared slug: feature-login.json + legacy := []byte(legacyRecordJSON(dashBranch, "myapp-preview-feature-login-v9")) + mock.Files[legacyPath] = legacy + seedCaddyfileWithRoute(mock, "myapp-preview-feature-login", "preview-feature-login.myapp.com") + + err := mgr.Deploy(context.Background(), deployCfg(loginBranch, "v1")) + var amb *AmbiguousPreviewError + if !errors.As(err, &amb) { + t.Fatalf("expected *AmbiguousPreviewError, got %v", err) + } + if amb.StoredBranch != dashBranch || amb.RequestedBranch != loginBranch { + t.Errorf("error must name both branches, got %+v", amb) + } + if amb.Path != legacyPath { + t.Errorf("error must name the record path %s, got %s", legacyPath, amb.Path) + } + for _, want := range []string{dashBranch, loginBranch, legacyPath} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error text must mention %q: %v", want, err) } } + + // Nothing was mutated: record intact, no container touched, no new + // record, no route edit, not even the preview directory. + if got := string(mock.Files[legacyPath]); got != string(legacy) { + t.Errorf("ambiguous legacy record was mutated:\nbefore: %s\nafter: %s", legacy, got) + } + if calls := mock.Calls; len(calls) > 2 { // the two record reads only + t.Errorf("ambiguous legacy record must abort before any mutation, calls: %v", calls) + } +} + +// Destroy hits the same ambiguity wall: it must refuse, not guess, and +// leave the record intact. +func TestLegacyCollisionDestroyIsAmbiguous(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + legacyPath := legacyPreviewStatePath("myapp", loginBranch) + legacy := []byte(legacyRecordJSON(dashBranch, "myapp-preview-feature-login-v9")) + mock.Files[legacyPath] = legacy + + err := mgr.Destroy(context.Background(), "myapp", loginBranch) + var amb *AmbiguousPreviewError + if !errors.As(err, &amb) { + t.Fatalf("expected *AmbiguousPreviewError, got %v", err) + } + if got := string(mock.Files[legacyPath]); got != string(legacy) { + t.Errorf("ambiguous legacy record was mutated:\nbefore: %s\nafter: %s", legacy, got) + } + if stops := callsContaining(mock, "docker"); len(stops) != 0 { + t.Errorf("no docker command may run against an ambiguous record, calls: %v", stops) + } + if rms := callsContaining(mock, "rm -f"); len(rms) != 0 { + t.Errorf("no file may be removed against an ambiguous record, calls: %v", rms) + } +} + +// Repo provenance participates in legacy adoption when both sides record +// one: same branch, different repo → ambiguous, untouched. +func TestLegacyRepoMismatchIsAmbiguous(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + legacyPath := legacyPreviewStatePath("myapp", loginBranch) + legacy := []byte(`{"branch":"feature/login","repo":"github.com/someone/clone","domain":"preview-feature-login.myapp.com","port":49200,"container":"myapp-preview-feature-login-v1","image":"myapp:v1","created_at":"2020-01-01T00:00:00Z","expires_at":"2099-01-01T00:00:00Z"}`) + mock.Files[legacyPath] = legacy + + cfg := deployCfg(loginBranch, "v1") // Repo github.com/tyler/myapp + err := mgr.Deploy(context.Background(), cfg) + var amb *AmbiguousPreviewError + if !errors.As(err, &amb) { + t.Fatalf("expected *AmbiguousPreviewError, got %v", err) + } + if !strings.Contains(err.Error(), "github.com/someone/clone") { + t.Errorf("error must name the stored repo: %v", err) + } + if got := string(mock.Files[legacyPath]); got != string(legacy) { + t.Errorf("ambiguous legacy record was mutated") + } +} + +// A legacy record for the OTHER colliding branch must not block work on +// this branch once this branch has its own canonical record: the legacy +// file belongs to that branch and is left in place untouched. +func TestLegacyOtherBranchNotBlocked(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + // This branch's canonical record exists (a modern deploy happened). + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + // And a legacy-era preview of the colliding branch is still around. + legacyPath := legacyPreviewStatePath("myapp", loginBranch) // feature-login.json + legacy := []byte(legacyRecordJSON(dashBranch, "myapp-preview-feature-login-v9")) + mock.Files[legacyPath] = legacy + + // Redeploying feature/login must succeed and not touch the other + // branch's legacy record or container. + mustDeploy(t, mgr, deployCfg(loginBranch, "v2")) + if got := string(mock.Files[legacyPath]); got != string(legacy) { + t.Errorf("the other branch's legacy record was mutated:\nbefore: %s\nafter: %s", legacy, got) + } + if len(callsContaining(mock, "docker stop -t 5 'myapp-preview-feature-login-v9'")) != 0 { + t.Errorf("the other branch's container must not be stopped: %v", callsContaining(mock, "docker stop")) + } + + // And destroying THAT branch still finds and removes its legacy record. + if err := mgr.Destroy(context.Background(), "myapp", dashBranch); err != nil { + t.Fatalf("Destroy of the legacy branch: %v", err) + } + if _, ok := mock.Files[legacyPath]; ok { + t.Errorf("legacy record for %s not removed by its own destroy", dashBranch) + } + if len(callsContaining(mock, "docker stop -t 5 'myapp-preview-feature-login-v9'")) != 1 { + t.Errorf("legacy branch's container not stopped via its stored name: %v", callsContaining(mock, "docker stop")) + } +} + +// List surfaces records from both eras without mutating anything. +func TestListIncludesLegacyAndCanonical(t *testing.T) { + loginPath := previewStatePath("myapp", loginBranch) + legacyPath := legacyPreviewStatePath("myapp", dashBranch) + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "ls /deployments/myapp/previews/*.json", + Output: loginPath + "\n" + legacyPath}, + ) + mock.Files[loginPath] = []byte(`{"id":"myapp-p-` + loginIDHex + `","branch":"feature/login","route":"myapp-preview-p-` + loginIDHex + `","domain":"preview-feature-login-` + loginIDHex + `.myapp.com","port":49200,"container":"myapp-preview-p-` + loginIDHex + `-v1","image":"myapp:v1","created_at":"2020-01-01T00:00:00Z","expires_at":"2099-01-01T00:00:00Z"}`) + mock.Files[legacyPath] = []byte(legacyRecordJSON(dashBranch, "myapp-preview-feature-login-v9")) + + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + previews, err := mgr.List(context.Background(), "myapp") + if err != nil { + t.Fatalf("List: %v", err) + } + if len(previews) != 2 { + t.Fatalf("expected 2 previews, got %d: %+v", len(previews), previews) + } + byBranch := map[string]State{} + for _, p := range previews { + byBranch[p.Branch] = p + } + if byBranch[loginBranch].ID != "myapp-p-"+loginIDHex { + t.Errorf("canonical record lost its ID: %+v", byBranch[loginBranch]) + } + if byBranch[dashBranch].ID != "" { + t.Errorf("legacy record must be listed unmodified (no invented ID): %+v", byBranch[dashBranch]) + } } From 01ec45cc4ecd0384e28f60efb1f7494d74b35112 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:54:16 -0700 Subject: [PATCH 08/12] fix(autodeploy,cli): scheduled redeploys run through the deploy engine (C02) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cron script's inspect-driven stop/rm/run container reconstruction (no lock, no health gate, no release record, no rollback, downtime window) is replaced by a digest pre-check plus invocation of the new server-side 'teploy autodeploy redeploy' — the same triggerAutoDeploy engine path the webhook listener uses. schedule gains --branch, uploads the server binary and capability-checks it; installed scripts upgrade on the next schedule run. Script-content tests assert the engine invocation and forbid docker run/stop/rm reconstruction; mutation check verified the guard. --- AUDIT_OPEN.md | 20 +++++ internal/autodeploy/autodeploy.go | 103 +++++++------------------ internal/autodeploy/autodeploy_test.go | 57 +++++++++++--- internal/cli/autodeploy.go | 83 ++++++++++++++++++-- 4 files changed, 170 insertions(+), 93 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 43dae69..96d13d7 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1052,3 +1052,23 @@ destroy-before-recreate stays as-is; expiry timer/automation beyond the existing deploy-piggyback prune; config propagation through a preview profile; network/secret isolation between previews) and any Dash-side changes. + +## Product programme slice (2026-09-22) — C02: scheduled redeploys run through the engine + +The scheduled-redeploy cron script reconstructed the container from +docker inspect and did its own stop/rm/run: no lock, no fence, no +health gate, no release record, no rollback, and a stop-to-start +downtime window — a second, weaker deploy path next to the engine +(C02's defect class; the script's own comment deferred this to "a v2"). +The script now performs only the cheap digest pre-check (no-op when +unchanged) and, when the digest moved, invokes the on-server teploy +binary's new `autodeploy redeploy` — the exact triggerAutoDeploy path +(fenced lock, fetch, config load, env resolution, health-gated deploy) +the webhook listener uses. `teploy autodeploy schedule` gained +--branch, uploads the server binary, and verifies it supports +`redeploy` before installing anything (actionable error until a +release carries it — v0.1.35 does NOT; first release with it must +precede rescheduling). Existing installed scripts keep the old +behavior until `schedule` is re-run. Webhook admission durability, +cancel/supersede policy and Dash/CI trigger convergence remain recorded +C02 scope. diff --git a/internal/autodeploy/autodeploy.go b/internal/autodeploy/autodeploy.go index d564ad8..2d20f39 100644 --- a/internal/autodeploy/autodeploy.go +++ b/internal/autodeploy/autodeploy.go @@ -281,7 +281,6 @@ func (m *Manager) allowWebhookPortInFirewall(ctx context.Context, sudo string, p return nil } - // SetupCaddyRoute persists the webhook route INTO THE CADDYFILE, inside // the app's managed site block (see internal/caddy/webhook.go — audit // T26/T27). The old runtime admin-API injection lived only in Caddy's @@ -355,13 +354,19 @@ func (m *Manager) ScheduleStatus(ctx context.Context, app string) (string, error // the image referenced by the currently-running container, compares digests, // and only recreates the container if the digest changed. No new image, no // container restart — quiet no-op. -func (m *Manager) Schedule(ctx context.Context, app, schedule string) error { +func (m *Manager) Schedule(ctx context.Context, app, schedule, branch, binaryPath string) error { if app == "" { return fmt.Errorf("app name is required") } if err := ValidateSchedule(schedule); err != nil { return err } + if err := ValidateBranch(branch); err != nil { + return err + } + if binaryPath == "" { + return fmt.Errorf("teploy binary path is required (the scheduled redeploy invokes the deploy engine on the server)") + } appDir := fmt.Sprintf("%s/%s", deploymentsDir, app) scriptPath := fmt.Sprintf("%s/%s", appDir, scheduledScriptName) @@ -371,7 +376,7 @@ func (m *Manager) Schedule(ctx context.Context, app, schedule string) error { } fmt.Fprintln(m.out, "Installing scheduled-redeploy script...") - script := generateScheduledRedeployScript(app) + script := generateScheduledRedeployScript(app, branch, binaryPath) if err := m.exec.Upload(ctx, strings.NewReader(script), scriptPath, "0755"); err != nil { return fmt.Errorf("uploading scheduled-redeploy script: %w", err) } @@ -397,7 +402,9 @@ func (m *Manager) Schedule(ctx context.Context, app, schedule string) error { fmt.Fprintf(m.out, "Scheduled redeploy installed for %s\n", app) fmt.Fprintf(m.out, " Schedule: %s\n", schedule) + fmt.Fprintf(m.out, " Branch: %s\n", branch) fmt.Fprintf(m.out, " Script: %s\n", scriptPath) + fmt.Fprintf(m.out, " Engine: %s autodeploy redeploy (locks, health gate, release record, rollback)\n", binaryPath) fmt.Fprintf(m.out, " Log: %s/scheduled-redeploy.log\n", appDir) return nil } @@ -481,25 +488,29 @@ func (m *Manager) Remove(ctx context.Context, app string) error { // // We keep the same container name on purpose: Caddy's reverse_proxy upstream // resolves containers by their network alias / DNS name, and re-using the -// name avoids a Caddy reconfigure step. The trade-off is a brief downtime -// window between stop and start (typically 1-3 seconds for Forgejo-class -// services). True zero-downtime swap belongs in a v2 that runs through -// teploy deploy on the server side. -func generateScheduledRedeployScript(app string) string { +// engine entry. The script only performs the CHEAP part — find the web +// container, pull its image tag, compare digests, and exit 0 when nothing +// changed. When the digest DID move it invokes the on-server teploy binary's +// `autodeploy redeploy`, which runs the exact same fenced, health-gated, +// release-recorded deploy code as `teploy deploy` and the webhook listener +// (C02: one execution path for every trigger). The old version of this +// script reconstructed the container from docker inspect and did its own +// stop/rm/run — no lock, no health gate, no release record, no rollback, +// and a stop-to-start downtime window. +func generateScheduledRedeployScript(app, branch, binaryPath string) string { return fmt.Sprintf(`#!/bin/bash -# Scheduled redeploy script for %[1]s -# Pulls the image and recreates the container if (and only if) the digest changed. +# Scheduled redeploy for %[1]s (branch %[2]s) — digest pre-check, then the full deploy engine. set -e -APP=%[1]q -PROCESS="web" +APP=%[4]q +BRANCH=%[5]q LOG="/deployments/$APP/scheduled-redeploy.log" ts() { date -u +%%Y-%%m-%%dT%%H:%%M:%%SZ; } -CONTAINER=$(docker ps --filter "label=teploy.app=$APP" --filter "label=teploy.process=$PROCESS" --format '{{.Names}}' | head -n 1) +CONTAINER=$(docker ps --filter "label=teploy.app=$APP" --filter "label=teploy.process=web" --format '{{.Names}}' | head -n 1) if [ -z "$CONTAINER" ]; then - echo "$(ts) [skip] no running container for $APP/$PROCESS" >> "$LOG" + echo "$(ts) [skip] no running container for $APP/web" >> "$LOG" exit 0 fi @@ -524,67 +535,9 @@ if [ "$CURRENT_DIGEST" = "$NEW_DIGEST" ]; then exit 0 fi -echo "$(ts) [redeploy] new digest for $IMAGE — recreating $CONTAINER" >> "$LOG" - -# Snapshot config from the running container before we tear it down. -ENV_FILE=$(mktemp) -docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$CONTAINER" > "$ENV_FILE" - -VOL_ARGS=() -while IFS= read -r line; do - [ -z "$line" ] && continue - VOL_ARGS+=("-v" "$line") -done < <(docker inspect --format='{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}:{{.Destination}} -{{else if eq .Type "bind"}}{{.Source}}:{{.Destination}} -{{end}}{{end}}' "$CONTAINER") - -LABEL_ARGS=() -NEW_VERSION=$(date +%%s) -while IFS= read -r line; do - [ -z "$line" ] && continue - KEY="${line%%%%=*}" - VAL="${line#*=}" - if [ "$KEY" = "teploy.version" ]; then - VAL="$NEW_VERSION" - fi - LABEL_ARGS+=("--label" "$KEY=$VAL") -done < <(docker inspect --format='{{range $k, $v := .Config.Labels}}{{$k}}={{$v}} -{{end}}' "$CONTAINER") - -PORT_ARGS=() -while IFS= read -r line; do - [ -z "$line" ] && continue - PORT_ARGS+=("-p" "$line") -done < <(docker inspect --format='{{range $port, $bindings := .NetworkSettings.Ports}}{{range $bindings}}{{.HostIp}}:{{.HostPort}}:{{$port}} -{{end}}{{end}}' "$CONTAINER" | sed 's|/tcp||;s|/udp||') - -NETWORK=$(docker inspect --format='{{range $n, $v := .NetworkSettings.Networks}}{{$n}}{{end}}' "$CONTAINER" | head -n 1) -RESTART=$(docker inspect --format='{{.HostConfig.RestartPolicy.Name}}' "$CONTAINER") -[ -z "$RESTART" ] && RESTART=no - -# Tear down the old container. -docker stop "$CONTAINER" >> "$LOG" 2>&1 || true -docker rm "$CONTAINER" >> "$LOG" 2>&1 || true - -# Recreate with the same name and config + new image. -NEW_ID=$(docker run -d \ - --name "$CONTAINER" \ - --network "${NETWORK:-bridge}" \ - --restart "$RESTART" \ - --env-file "$ENV_FILE" \ - "${LABEL_ARGS[@]}" \ - "${VOL_ARGS[@]}" \ - "${PORT_ARGS[@]}" \ - "$IMAGE" 2>>"$LOG") || { - echo "$(ts) [error] docker run failed — container is down" >> "$LOG" - rm -f "$ENV_FILE" - exit 1 - } - -rm -f "$ENV_FILE" - -echo "$(ts) [ok] $CONTAINER redeployed (id=${NEW_ID:0:12} version=$NEW_VERSION)" >> "$LOG" -`, app) +echo "$(ts) [redeploy] new digest for $IMAGE — running the deploy engine" >> "$LOG" +%[3]q autodeploy redeploy --app "$APP" --branch "$BRANCH" >> "$LOG" 2>&1 +`, app, branch, binaryPath, app, branch) } // generateService renders the systemd unit that runs execStart (the full diff --git a/internal/autodeploy/autodeploy_test.go b/internal/autodeploy/autodeploy_test.go index 1ca8e1f..26ada65 100644 --- a/internal/autodeploy/autodeploy_test.go +++ b/internal/autodeploy/autodeploy_test.go @@ -445,7 +445,7 @@ func TestSchedule(t *testing.T) { var buf bytes.Buffer mgr := NewManager(mock, &buf) - err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0") + err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0", "main", "/deployments/.bin/teploy") if err != nil { t.Fatalf("Schedule: %v", err) } @@ -454,19 +454,28 @@ func TestSchedule(t *testing.T) { if !ok { t.Fatal("scheduled-redeploy.sh not uploaded") } + // The C02 contract: the cheap digest pre-check stays, but the actual + // redeploy runs through the on-server engine — never a script-side + // container reconstruction (no docker run/stop/rm). for _, want := range []string{ `APP="myapp"`, + `BRANCH="main"`, "docker pull", "docker inspect", "teploy.app=$APP", - "teploy.process=$PROCESS", "CURRENT_DIGEST", "NEW_DIGEST", + `"/deployments/.bin/teploy" autodeploy redeploy --app "$APP" --branch "$BRANCH"`, } { if !strings.Contains(string(script), want) { t.Errorf("scheduled-redeploy.sh missing %q", want) } } + for _, forbidden := range []string{"docker run", "docker stop", "docker rm"} { + if strings.Contains(string(script), forbidden) { + t.Errorf("scheduled-redeploy.sh must not reconstruct containers itself (%q present) — the redeploy goes through the engine", forbidden) + } + } if !strings.Contains(buf.String(), "Scheduled redeploy installed for myapp") { t.Error("expected install confirmation") @@ -474,13 +483,16 @@ func TestSchedule(t *testing.T) { if !strings.Contains(buf.String(), "0 4 * * 0") { t.Error("expected cron schedule in output") } + if !strings.Contains(buf.String(), "main") { + t.Error("expected branch in output") + } } func TestSchedule_RejectsBadCron(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4") mgr := NewManager(mock, &bytes.Buffer{}) - err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0; echo pwned") + err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0; echo pwned", "main", "/deployments/.bin/teploy") if err == nil { t.Fatal("expected validation error for shell-metachar cron string") } @@ -490,7 +502,7 @@ func TestSchedule_RejectsEmptyApp(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4") mgr := NewManager(mock, &bytes.Buffer{}) - if err := mgr.Schedule(context.Background(), "", "0 4 * * 0"); err == nil { + if err := mgr.Schedule(context.Background(), "", "0 4 * * 0", "main", "/deployments/.bin/teploy"); err == nil { t.Fatal("expected error for empty app name") } } @@ -541,22 +553,45 @@ func TestScheduleStatus_Active(t *testing.T) { } func TestGenerateScheduledRedeployScript(t *testing.T) { - script := generateScheduledRedeployScript("myapp") + script := generateScheduledRedeployScript("myapp", "release", "/deployments/.bin/teploy") for _, want := range []string{ `APP="myapp"`, - `PROCESS="web"`, + `BRANCH="release"`, "docker pull", - "teploy.version", "docker inspect", - "docker run -d", - "--name \"$CONTAINER\"", // preserve same name to avoid Caddy reconfig - "date +%s", // new version timestamp + "CURRENT_DIGEST", + "NEW_DIGEST", + `"/deployments/.bin/teploy" autodeploy redeploy --app "$APP" --branch "$BRANCH"`, "$(ts) [redeploy]", } { if !strings.Contains(script, want) { t.Errorf("scheduled redeploy script missing %q", want) } } + // C02: the script must never reconstruct the container itself — the + // redeploy runs through the engine (locks, health gate, records). + for _, forbidden := range []string{"docker run", "docker stop", "docker rm"} { + if strings.Contains(script, forbidden) { + t.Errorf("scheduled redeploy script must not contain %q — container reconstruction bypasses the engine", forbidden) + } + } +} + +// TestSchedule_RejectsBadBranch: the branch lands inside a server-side +// script, so an invalid one is refused before anything is installed. +func TestSchedule_RejectsBadBranch(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4") + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + if err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0", "main; rm -rf /", "/deployments/.bin/teploy"); err == nil { + t.Fatal("expected an error for a branch with shell metacharacters") + } + if err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0", "", "/deployments/.bin/teploy"); err == nil { + t.Fatal("expected an error for an empty branch") + } + if err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0", "main", ""); err == nil { + t.Fatal("expected an error for an empty binary path") + } } // TestSchedule_FailedCrontabReadAborts is the T29 regression: a failed @@ -570,7 +605,7 @@ func TestSchedule_FailedCrontabReadAborts(t *testing.T) { ) var buf bytes.Buffer mgr := NewManager(mock, &buf) - if err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0"); err == nil { + if err := mgr.Schedule(context.Background(), "myapp", "0 4 * * 0", "main", "/deployments/.bin/teploy"); err == nil { t.Fatal("a failed crontab read must abort the install, never replace the crontab") } for _, c := range mock.Calls { diff --git a/internal/cli/autodeploy.go b/internal/cli/autodeploy.go index d57d287..cdf275e 100644 --- a/internal/cli/autodeploy.go +++ b/internal/cli/autodeploy.go @@ -29,6 +29,7 @@ func newAutoDeployCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newAutoDeployScheduleCmd(flags)) cmd.AddCommand(newAutoDeployUnscheduleCmd(flags)) cmd.AddCommand(newAutoDeployServeCmd()) + cmd.AddCommand(newAutoDeployRedeployCmd()) return cmd } @@ -269,24 +270,32 @@ func newAutoDeployScheduleCmd(flags *Flags) *cobra.Command { Use: "schedule ", Short: "Schedule periodic redeploys to refresh the image", Long: `Installs a cron job on the server that periodically pulls the image -referenced by the running container and redeploys only if a newer -digest is available. No-op when the image is already current. +referenced by the running container and, only when a newer digest is +available, redeploys through the full teploy engine — the same fenced, +health-gated, release-recorded deploy as ` + "`teploy deploy`" + ` and the +webhook listener (one execution path for every trigger). No-op when +the image is already current. Use this when the image tag is pinned to a major version (e.g. :14) and you want to receive its patch releases automatically. +The server needs a teploy binary that supports ` + "`autodeploy redeploy`" + `; +this command installs one and verifies it. + Examples: - teploy autodeploy schedule "0 4 * * 0" # Sundays at 4am - teploy autodeploy schedule "0 */6 * * *" # every 6 hours`, + teploy autodeploy schedule "0 4 * * 0" # Sundays at 4am, branch main + teploy autodeploy schedule --branch release "0 */6 * * *" # every 6 hours`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runAutoDeploySchedule(flags, args[0]) + branch, _ := cmd.Flags().GetString("branch") + return runAutoDeploySchedule(flags, args[0], branch) }, } + cmd.Flags().String("branch", "main", "branch the scheduled redeploy fetches and deploys") return cmd } -func runAutoDeploySchedule(flags *Flags, schedule string) error { +func runAutoDeploySchedule(flags *Flags, schedule, branch string) error { appCfg, err := config.LoadApp(".") if err != nil { return err @@ -294,6 +303,9 @@ func runAutoDeploySchedule(flags *Flags, schedule string) error { if err := autodeploy.ValidateSchedule(schedule); err != nil { return err } + if err := autodeploy.ValidateBranch(branch); err != nil { + return err + } ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() @@ -304,8 +316,65 @@ func runAutoDeploySchedule(flags *Flags, schedule string) error { } defer executor.Close() + // The scheduled script triggers the on-server deploy engine rather + // than reconstructing the container itself (C02), so the server needs + // a teploy binary that speaks `autodeploy redeploy`. + const teployBinaryPath = "/deployments/.bin/teploy" + if _, err := deployTeployBinaryToServer(ctx, executor, teployBinaryPath); err != nil { + return fmt.Errorf("installing the teploy binary on the server: %w", err) + } + if _, err := executor.Run(ctx, fmt.Sprintf("%s autodeploy redeploy --help >/dev/null 2>&1", ssh.ShellQuote(teployBinaryPath))); err != nil { + return fmt.Errorf("the server's teploy binary does not support 'autodeploy redeploy' (the scheduled redeploy now runs the full engine through it); release a teploy version that includes it, then re-run this command: %w", err) + } + mgr := autodeploy.NewManager(executor, os.Stdout) - return mgr.Schedule(ctx, appCfg.App, schedule) + return mgr.Schedule(ctx, appCfg.App, schedule, branch, teployBinaryPath) +} + +// newAutoDeployRedeployCmd is the one-shot engine trigger the scheduled +// redeploy invokes ON THE SERVER. It runs the exact same fenced, +// health-gated, release-recorded deploy code as `teploy deploy` and the +// webhook listener's triggerAutoDeploy — C02's one execution path. It +// needs no webhook listener, no secret and no local checkout; the +// server-side build directory (created by `teploy deploy`'s server-build +// mode or `autodeploy setup`) is fetched to the branch tip before +// deploying. +func newAutoDeployRedeployCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "redeploy", + Short: "One-shot engine deploy (server-side; used by the scheduled redeploy)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + app, _ := cmd.Flags().GetString("app") + branch, _ := cmd.Flags().GetString("branch") + strictEnv, _ := cmd.Flags().GetBool("strict-env") + return runAutoDeployRedeploy(app, branch, strictEnv) + }, + } + cmd.Flags().String("app", "", "app name (required)") + cmd.Flags().String("branch", "main", "branch to fetch and deploy") + cmd.Flags().Bool("strict-env", false, "fail the deploy when env: references an unset ${VAR} (also enabled by TEPLOY_STRICT_ENV=1)") + return cmd +} + +func runAutoDeployRedeploy(app, branch string, strictEnv bool) error { + if err := config.ValidateName(app); err != nil { + return err + } + if err := autodeploy.ValidateBranch(branch); err != nil { + return err + } + if !strictEnv && os.Getenv("TEPLOY_STRICT_ENV") == "1" { + strictEnv = true + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + executor := ssh.NewLocalExecutor() + defer executor.Close() + + return triggerAutoDeploy(ctx, executor, app, branch, autodeploy.BuildDir(app), os.Stdout, nil, false, strictEnv) } func newAutoDeployUnscheduleCmd(flags *Flags) *cobra.Command { From c24a72d762962c4e3c71a8c264609796c537e2c3 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:19:48 -0700 Subject: [PATCH 09/12] fix(autodeploy): durable webhook admission before ack, bounded newest-wins queue (C02) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recon found the admission contract worse than recorded: a delivery arriving during a running deploy was acked 200 then silently dropped (AcquireLockFenced returns immediately, it does not block) — every push during a deploy was lost. Now: an append-only fsync'd admission ledger (.autodeploy-ledger.jsonl, 0600) records admitted/superseded/processed BEFORE the 200 (mutation-verified ordering); one worker + one newest-wins pending slot supersedes older pending deliveries instead of piling goroutines; serve restart resumes admitted-but-unprocessed work (newest per app, dedup-reseeded); ledger persistence failure is 503 + Retry-After with the dedup entry rolled back. Commit-pinned builds (fetch pins to branch tip, not payload commit) remain recorded C02 scope. --- AUDIT_OPEN.md | 100 ++++ internal/autodeploy/ledger.go | 236 +++++++++ internal/autodeploy/ledger_test.go | 125 +++++ internal/autodeploy/webhook.go | 22 + internal/cli/autodeploy_admission_test.go | 551 ++++++++++++++++++++++ internal/cli/autodeploy_serve.go | 339 +++++++++++-- internal/cli/autodeploy_serve_test.go | 398 ++++++++++------ 7 files changed, 1568 insertions(+), 203 deletions(-) create mode 100644 internal/autodeploy/ledger.go create mode 100644 internal/autodeploy/ledger_test.go create mode 100644 internal/cli/autodeploy_admission_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 96d13d7..66922f6 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1072,3 +1072,103 @@ precede rescheduling). Existing installed scripts keep the old behavior until `schedule` is re-run. Webhook admission durability, cancel/supersede policy and Dash/CI trigger convergence remain recorded C02 scope. + +## Product programme slice (2026-09-22, later) — C02: webhook admission durability + +The WEBHOOK trigger's admission contract (C02: "durable before +acknowledgment, bound to the authenticated commit, bounded queueing, +deduplication, cancel/supersede policy"). Base revision `01ec45c`; changes +left uncommitted for review. Closes the A34/T24 durable-webhook-queue +defect and the bounded-admission half of A37/T33 for this trigger path. + +**Recon (what was wrong, file:line at base):** the handler verified HMAC, +checked in-memory dedup (persisted best-effort), and sent 200 with the +trigger merely STARTED — `internal/cli/autodeploy_serve.go:293` acked +before anything durable existed; the dedup tmp+rename at :134-141 was +synchronous but errors were swallowed (a 200 could go out with nothing on +disk). Worse than pileup: the per-delivery goroutine (:149) called +`triggerAutoDeploy` → `AcquireLockFenced` (:332), which does NOT block — +`acquireAutoLock` (state.go:479-516) returns "deploy is already in +progress" immediately, so **every delivery arriving during a running +deploy was acked 200 and then silently dropped** (unbounded short-lived +goroutine spawn, zero queueing). The dedup file records +`map["content:"+sha256(body)]time.Time` — body digest only, delivery ID +is log metadata (A36). A serve restart lost every acked-but-unprocessed +admission (no record existed). + +**Landed:** + +- **Admission ledger** — `internal/autodeploy/ledger.go`: append-only + JSONL at `/deployments//.autodeploy-ledger.jsonl` (0600, next to + the dedup file), one record per line (admitted / superseded / processed + carrying id + provider delivery id + authenticated body digest + app + + branch + received-at), `FileLedger.Append` = single write + fsync (the + sibling+fsync+rename discipline applies to whole-file replacement; an + append-only log durably appends). `ParseLedger` ignores a torn FINAL + line (crash mid-append = never fsynced = never acked) and fails closed + on mid-file corruption or unknown kinds; `FoldAdmissions` folds to + pending + digest map; `NewestPending` picks newest per app. +- **Ack after fsync** — the handler appends the admission record and only + then writes 200 `{"status":"admitted","disposition":…}`; a persistence + failure rolls the dedup entry back (`DeliveryDedup.Unrecord` — the + provider's retry of the same signed body re-runs admission instead of + being swallowed as a replay of something never admitted) and answers + 503 + Retry-After: never ack what isn't durable. Replays answer 200 + `{"status":"duplicate"}`. The dedup snapshot persist + (`onDedupChanged`) moved to AFTER durable admission, so a dedup entry + on disk always corresponds to a ledger admission (a ping/tag persisting + dedup it never admitted was the subtle loss window; non-push acks are + now memory-only — their post-restart replay is a harmless no-op ack). +- **Bounded queue with supersede** — `admissionQueue` + (autodeploy_serve.go): pending work is a RECORD, never a blocked + goroutine. One worker (the only deploy runner — replaces the + fire-and-forget goroutine), one newest-wins pending slot: idle → + `running`; worker busy + slot empty → `queued`; slot filled → the older + pending is marked superseded in the ledger (by-id) and replaced + (`superseded` disposition). The RUNNING deploy is never cancelled + mid-flight (cancellation propagation is deliberately out of scope; the + newest deploy runs next instead). Same-digest redelivery while queued = + dedup path (`duplicate`). +- **Restart resume** — `resumeAdmissions` on serve start: fold the + ledger, reseed the replay dedup from recent admitted digests (the + ledger backstops the best-effort dedup file across the delivery TTL), + mark older pendings superseded, admit the newest per app with the + changed-file list unrecoverable → filesKnown=false (deploy fail-open, + the documented monorepo rule). Processed entries never re-trigger; + duplicate delivery ids / digests collapse via newest-wins. + +**Evidence** — TDD red against the base handler (both recorded failing): +10 distinct-body deliveries → 10 trigger calls (want ≤2), and the +admission response carried no disposition. Green after the rework. +Mutation check: moving the 200 ahead of the ledger append fails +`TestAdmission_AckWaitsForFsync` ("response written (status 200) before +the admission fsync completed"); reverted. New coverage: ack-after-fsync +ordering (gated ledger + response-flagging writer), persistence-failure +→ 503 + nothing admitted + retry-after-recovery admitted, supersede (A +marked superseded-by-B in the ledger, only B runs after the running +deploy), 10-delivery pileup collapse (1 running / 1 queued / 8 +superseded; exactly 2 deploy invocations), crash-resume (exactly one +re-trigger; processed never re-triggers; newest-wins over duplicate +delivery ids; dedup reseeded), FileLedger concurrency/durability, +torn-tail/corruption parsing, fold/newest/seed units. Gates: `go vet +./...` clean; `go test ./... -race` all 25 packages ok; gofmt clean on +touched files (deploy.go/secret_audit.go/update_test.go were unformatted +at base — left alone). + +**Remaining C02 scope (explicitly NOT in this slice):** + +- **Commit-pinned builds — still open, verified**: `triggerAutoDeploy` + fetches and resets to `origin/` TIP (autodeploy_serve.go + `git fetch origin && git reset --hard origin/`), not the + authenticated payload's `after` commit — the tip IS pinned to the + remote ref at deploy time, but a push landing between event and fetch + deploys the newer commit under the older event's admission. Full + pinning = F40/A35/T25 (fetch + worktree checkout of the event commit). +- **Dash/CI trigger convergence** — teploy-dash and CI-triggered deploys + do not go through this admission path; converging them onto the ledger + + queue (or the engine trigger generally) is cross-repo work. +- **Cancellation propagation** — a supersede never interrupts a running + deploy; the newest runs next. Mid-deploy cancel is a deliberate + non-goal here (the register's stranding posture) and stays open with + the graceful-shutdown/bounded-admission remainder of A37/T33 (listener + scope, signal-time drain of the worker). diff --git a/internal/autodeploy/ledger.go b/internal/autodeploy/ledger.go new file mode 100644 index 0000000..3733cb3 --- /dev/null +++ b/internal/autodeploy/ledger.go @@ -0,0 +1,236 @@ +package autodeploy + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strings" + "time" +) + +// The webhook admission ledger (C02): an append-only record of every +// delivery this listener durably admitted, so the 200 a provider receives +// is backed by bytes on disk that survive a crash immediately after the +// response. One JSON record per line at /deployments// +// .autodeploy-ledger.jsonl (the serve process runs on the server itself, +// like the dedup file next to it). +// +// Record kinds: +// +// admitted — a verified push to the watched branch was accepted +// superseded — a pending (not yet running) admission was replaced by a +// newer delivery; superseded_by names the replacement +// processed — the deploy the admission caused ran to completion +// +// Folding the file yields the admitted-but-never-terminal records: on +// restart the serve process replays exactly those (newest per app wins), +// which is the correct webhook contract — unlike a UI admission write, the +// provider will NOT redeliver an event it was told was accepted. + +// Admission record kinds (see the ledger commentary above). +const ( + AdmissionKindAdmitted = "admitted" + AdmissionKindSuperseded = "superseded" + AdmissionKindProcessed = "processed" +) + +// AdmissionRecord is one line of the admission ledger. The identity fields +// (ID/Digest/App/Branch/Received) are carried on every record so a folded +// line is self-describing; transition kinds add their own fields. +type AdmissionRecord struct { + Kind string `json:"kind"` + ID string `json:"id"` + Delivery string `json:"delivery,omitempty"` // provider delivery header, metadata only (A36) + Digest string `json:"digest"` // hex sha256 of the AUTHENTICATED body + App string `json:"app"` + Branch string `json:"branch,omitempty"` + Received time.Time `json:"received"` + SupersededBy string `json:"superseded_by,omitempty"` + At time.Time `json:"at,omitempty"` // transition time (superseded/processed) +} + +// NewAdmissionID mints a unique id for an admission record. +func NewAdmissionID() string { + var id [16]byte + if _, err := rand.Read(id[:]); err == nil { + return hex.EncodeToString(id[:]) + } + fallback := sha256.Sum256([]byte(time.Now().UTC().Format(time.RFC3339Nano))) + return hex.EncodeToString(fallback[:16]) +} + +// ContentID is the dedup/ledger key for an authenticated body: the same +// digest the DeliveryDedup records, derived here so resume can seed dedup +// from ledger records without re-reading bodies. +func ContentID(body []byte) string { + sum := sha256.Sum256(body) + return "content:" + hex.EncodeToString(sum[:]) +} + +// ContentIDFromDigest is ContentID for a digest already computed (ledger +// records store the bare hex digest). +func ContentIDFromDigest(digestHex string) string { + return "content:" + digestHex +} + +// LedgerAppender durably appends one record: when Append returns nil, the +// record is on disk and fsynced — that is the admission durability point +// the webhook handler acks against. The interface is the test seam for +// ordering and failure injection. +type LedgerAppender interface { + Append(rec AdmissionRecord) error +} + +// FileLedger is the on-disk append-only ledger. Opens the file +// O_APPEND|O_CREATE (0600 — same privacy class as the dedup file) and +// fsyncs after every appended line. +type FileLedger struct { + path string + mu chan struct{} // binary semaphore; keeps Append a mutex-free test target + f *os.File +} + +// OpenLedger opens (or creates) the ledger at path for durable appends. +func OpenLedger(path string) (*FileLedger, error) { + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return nil, fmt.Errorf("opening webhook admission ledger %s: %w", path, err) + } + return &FileLedger{path: path, mu: make(chan struct{}, 1), f: f}, nil +} + +// Append writes one record as a single line and fsyncs before returning. +// A non-nil error means the record MUST be treated as not durable. +func (l *FileLedger) Append(rec AdmissionRecord) error { + line, err := json.Marshal(rec) + if err != nil { + return fmt.Errorf("encoding admission record: %w", err) + } + line = append(line, '\n') + + l.mu <- struct{}{} + _, werr := l.f.Write(line) + if werr == nil { + werr = l.f.Sync() + } + <-l.mu + if werr != nil { + return fmt.Errorf("appending to webhook admission ledger %s: %w", l.path, werr) + } + return nil +} + +// Close closes the underlying file. +func (l *FileLedger) Close() error { + if l == nil || l.f == nil { + return nil + } + return l.f.Close() +} + +// ErrLedgerCorrupt flags a ledger line that cannot be parsed mid-file — +// the fold refuses to guess around unknown history. +var ErrLedgerCorrupt = errors.New("webhook admission ledger corrupt") + +// ParseLedger parses raw ledger bytes into records. A torn FINAL line (no +// trailing newline — a crash mid-append, never fsynced, never acked) is +// ignored; any other unparseable line fails with ErrLedgerCorrupt. +func ParseLedger(data []byte) ([]AdmissionRecord, error) { + if len(data) == 0 { + return nil, nil + } + torn := len(data) > 0 && data[len(data)-1] != '\n' + lines := strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") + if torn { + lines = lines[:len(lines)-1] + } + recs := make([]AdmissionRecord, 0, len(lines)) + for i, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + var rec AdmissionRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return nil, fmt.Errorf("%w: %s line %d: %v", ErrLedgerCorrupt, "ledger", i+1, err) + } + switch rec.Kind { + case AdmissionKindAdmitted, AdmissionKindSuperseded, AdmissionKindProcessed: + default: + return nil, fmt.Errorf("%w: unknown record kind %q at line %d", ErrLedgerCorrupt, rec.Kind, i+1) + } + recs = append(recs, rec) + } + return recs, nil +} + +// FoldedAdmission is the folded state of the ledger: the records still +// admitted-and-unprocessed, in admission order. +type FoldedAdmission struct { + Record AdmissionRecord +} + +// FoldAdmissions folds parsed records into (a) the pending admissions — +// admitted, never superseded, never processed, in admission order — and +// (b) every admitted digest with its received time, so restart can reseed +// the replay dedup from the durable ledger even when the best-effort +// dedup file lost an entry. +func FoldAdmissions(recs []AdmissionRecord) (pending []AdmissionRecord, digests map[string]time.Time) { + digests = make(map[string]time.Time) + state := make(map[string]string) // id -> admitted|superseded|processed + for _, rec := range recs { + switch rec.Kind { + case AdmissionKindAdmitted: + state[rec.ID] = AdmissionKindAdmitted + if _, ok := digests[rec.Digest]; !ok { + digests[rec.Digest] = rec.Received + } + case AdmissionKindSuperseded, AdmissionKindProcessed: + if prev, ok := state[rec.ID]; ok && prev == AdmissionKindAdmitted { + state[rec.ID] = rec.Kind + } + } + } + for _, rec := range recs { + if rec.Kind == AdmissionKindAdmitted && state[rec.ID] == AdmissionKindAdmitted { + pending = append(pending, rec) + } + } + return pending, digests +} + +// NewestPending returns the newest pending admission for app from a folded +// pending list (the last admitted-unprocessed record for that app), or nil. +func NewestPending(pending []AdmissionRecord, app string) *AdmissionRecord { + var newest *AdmissionRecord + for i := range pending { + if pending[i].App == app && (newest == nil || !pending[i].Received.Before(newest.Received)) { + p := pending[i] + newest = &p + } + } + return newest +} + +// SeedDedupFromLedger records recent admitted digests into a dedup set so +// a redelivery after a restart is rejected as the replay it is even when +// the best-effort dedup file never persisted the entry. Entries older than +// the delivery TTL are skipped (matching the dedup prune window). +func SeedDedupFromLedger(d *DeliveryDedup, digests map[string]time.Time, now time.Time) { + var keys []string + for k := range digests { + keys = append(keys, k) + } + sort.Strings(keys) + for _, digest := range keys { + t := digests[digest] + if now.Sub(t) > deliveryTTL { + continue + } + d.RecordOnly(ContentIDFromDigest(digest)) + } +} diff --git a/internal/autodeploy/ledger_test.go b/internal/autodeploy/ledger_test.go new file mode 100644 index 0000000..41cb5f5 --- /dev/null +++ b/internal/autodeploy/ledger_test.go @@ -0,0 +1,125 @@ +package autodeploy + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestDeliveryDedup_UnrecordRollsBackFailedAdmission(t *testing.T) { + d := NewDeliveryDedup() + if d.SeenAndRecord("content:abc") { + t.Fatal("first record reported as replay") + } + // The admission this record fronted failed to persist: unrecord so the + // provider retry goes through admission again. + d.Unrecord("content:abc") + if d.SeenAndRecord("content:abc") { + t.Fatal("after unrecord the digest still reads as replay — a failed admission would swallow the retry") + } +} + +func TestDeliveryDedup_RecordOnlySeedsWithoutReplayFlag(t *testing.T) { + d := NewDeliveryDedup() + d.RecordOnly("content:seed") + if !d.SeenAndRecord("content:seed") { + t.Fatal("seeded digest must read as replay") + } + if d.SeenAndRecord("content:other") { + t.Fatal("unrelated digest must not read as replay") + } +} + +func TestContentIDGolden(t *testing.T) { + got := ContentID([]byte(`{"ref":"refs/heads/main"}`)) + // Pin the derivation (sha256 hex with the content: prefix) without + // hand-computing a constant: derive once, ensure it is stable and hex. + if got == "" || len(got) != len("content:")+64 { + t.Fatalf("ContentID = %q, want content:<64 hex chars>", got) + } + if got != ContentIDFromDigest(got[len("content:"):]) { + t.Fatal("ContentID and ContentIDFromDigest disagree on the same digest") + } +} + +func TestNewAdmissionIDUnique(t *testing.T) { + seen := make(map[string]bool) + for i := 0; i < 1000; i++ { + id := NewAdmissionID() + if seen[id] { + t.Fatalf("duplicate admission id %q", id) + } + seen[id] = true + } +} + +func TestFileLedgerAppendsAreDurableAndSerialized(t *testing.T) { + path := filepath.Join(t.TempDir(), "ledger.jsonl") + l, err := OpenLedger(path) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 32) + for i := 0; i < 32; i++ { + go func(n int) { + done <- l.Append(AdmissionRecord{Kind: AdmissionKindAdmitted, ID: NewAdmissionID(), Digest: "d", App: "app"}) + }(i) + } + for i := 0; i < 32; i++ { + if err := <-done; err != nil { + t.Fatalf("concurrent append: %v", err) + } + } + l.Close() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + recs, err := ParseLedger(data) + if err != nil { + t.Fatal(err) + } + if len(recs) != 32 { + t.Fatalf("parsed %d records, want 32 (interleaved writes must not corrupt lines)", len(recs)) + } +} + +func TestFoldAdmissionsPendingOrderAndDigests(t *testing.T) { + now := time.Now().UTC() + recs := []AdmissionRecord{ + {Kind: AdmissionKindAdmitted, ID: "a", Digest: "da", App: "app", Received: now}, + {Kind: AdmissionKindAdmitted, ID: "b", Digest: "db", App: "app", Received: now.Add(time.Second)}, + {Kind: AdmissionKindSuperseded, ID: "a", Digest: "da", App: "app", SupersededBy: "b"}, + {Kind: AdmissionKindAdmitted, ID: "c", Digest: "dc", App: "other", Received: now.Add(2 * time.Second)}, + } + pending, digests := FoldAdmissions(recs) + if len(pending) != 2 || pending[0].ID != "b" || pending[1].ID != "c" { + t.Fatalf("pending = %+v, want [b c] in admission order", pending) + } + if len(digests) != 3 { + t.Fatalf("digests = %d, want 3", len(digests)) + } + if newest := NewestPending(pending, "app"); newest == nil || newest.ID != "b" { + t.Fatalf("NewestPending(app) = %+v, want b", newest) + } + if newest := NewestPending(pending, "missing"); newest != nil { + t.Fatalf("NewestPending(unknown app) = %+v, want nil", newest) + } +} + +func TestSeedDedupFromLedgerHonorsTTL(t *testing.T) { + d := NewDeliveryDedup() + now := time.Now().UTC() + digests := map[string]time.Time{ + "fresh": now.Add(-time.Hour), + "stale": now.Add(-deliveryTTL - time.Hour), + } + SeedDedupFromLedger(d, digests, now) + if !d.SeenAndRecord(ContentIDFromDigest("fresh")) { + t.Error("recent admitted digest must be seeded as replay") + } + if d.SeenAndRecord(ContentIDFromDigest("stale")) { + t.Error("digest older than the delivery TTL must not be seeded") + } +} diff --git a/internal/autodeploy/webhook.go b/internal/autodeploy/webhook.go index a64181f..3280d33 100644 --- a/internal/autodeploy/webhook.go +++ b/internal/autodeploy/webhook.go @@ -88,6 +88,28 @@ func (d *DeliveryDedup) SeenAndRecord(id string) bool { return false } +// RecordOnly records id as seen without reporting (used when reseeding the +// dedup set from the durable admission ledger on restart — those events +// were already admitted, so their digests must read as replays). +func (d *DeliveryDedup) RecordOnly(id string) { + d.mu.Lock() + defer d.mu.Unlock() + if _, ok := d.seen[id]; !ok { + d.seen[id] = time.Now() + } +} + +// Unrecord removes id from the seen set — the rollback half of an +// admission that failed to persist: the provider will retry the same +// signed body, and that retry must go through admission again instead of +// being swallowed as a replay of something that was never durably +// admitted (C02). +func (d *DeliveryDedup) Unrecord(id string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.seen, id) +} + // Snapshot returns a JSON-serializable copy for persisting across process // restarts (see LoadDeliveryDedup). func (d *DeliveryDedup) Snapshot() ([]byte, error) { diff --git a/internal/cli/autodeploy_admission_test.go b/internal/cli/autodeploy_admission_test.go new file mode 100644 index 0000000..e3fca16 --- /dev/null +++ b/internal/cli/autodeploy_admission_test.go @@ -0,0 +1,551 @@ +package cli + +// C02 webhook admission durability tests: ack-after-persist ordering, +// crash-between-admit-and-trigger resume, supersede, bounded queueing, and +// persistence-failure refusal. + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/useteploy/teploy/internal/autodeploy" +) + +// recordingWriter flags the instant a response status/body write begins — +// the observable for "was the response written before the fsync?". +type recordingWriter struct { + header http.Header + responded atomic.Bool + code atomic.Int32 +} + +func newRecordingWriter() *recordingWriter { + return &recordingWriter{header: make(http.Header)} +} + +func (w *recordingWriter) Header() http.Header { return w.header } + +func (w *recordingWriter) WriteHeader(code int) { + w.responded.Store(true) + w.code.Store(int32(code)) +} + +func (w *recordingWriter) Write(p []byte) (int, error) { + w.responded.Store(true) + return len(p), nil +} + +// gatedLedger is a LedgerAppender whose append passes through an explicit +// write → fsync gate: the "write" phase records the entry, then the append +// blocks in its "fsync" until released. Acks observed while the fsync is +// still blocked are acks-before-durability. +type gatedLedger struct { + mu sync.Mutex + records []autodeploy.AdmissionRecord + writeDone chan struct{} + fsyncStart chan struct{} + fsyncDone chan struct{} + releaseOnce sync.Once +} + +func newGatedLedger() *gatedLedger { + return &gatedLedger{ + writeDone: make(chan struct{}, 1), + fsyncStart: make(chan struct{}, 1), + fsyncDone: make(chan struct{}, 1), + } +} + +func (g *gatedLedger) Append(rec autodeploy.AdmissionRecord) error { + g.mu.Lock() + g.records = append(g.records, rec) + g.mu.Unlock() + g.writeDone <- struct{}{} + g.fsyncStart <- struct{}{} // entering fsync; blocking there until release + <-g.fsyncDone + return nil +} + +func (g *gatedLedger) releaseFsync() { + g.releaseOnce.Do(func() { close(g.fsyncDone) }) +} + +// TestAdmission_AckWaitsForFsync (C02 3a): the 200 must not be written +// before the admission ledger append (write + fsync) completes. +func TestAdmission_AckWaitsForFsync(t *testing.T) { + run := newCountingRun() + ledger := newGatedLedger() + queue := newAdmissionQueue(ledger, run.run, func(string, ...any) {}) + handler := newWebhookHandler(webhookHandlerConfig{ + secret: "s3cret", + branch: "main", + app: "myapp", + dedup: autodeploy.NewDeliveryDedup(), + ledger: ledger, + queue: queue, + logf: func(string, ...any) {}, + }) + + body := `{"ref":"refs/heads/main","after":"abc"}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("X-Hub-Signature-256", githubSign("s3cret", []byte(body))) + w := newRecordingWriter() + + handlerDone := make(chan struct{}) + go func() { + handler(w, req) + close(handlerDone) + }() + + select { + case <-ledger.writeDone: + case <-time.After(5 * time.Second): + t.Fatal("ledger append never started") + } + select { + case <-ledger.fsyncStart: + case <-handlerDone: + t.Fatal("handler finished without ever reaching the ledger fsync — admission was never durable") + } + + // The fsync is blocked NOW. If the response has been written at this + // point, the ack preceded durability. + if w.responded.Load() { + t.Fatalf("response written (status %d) before the admission fsync completed", w.code.Load()) + } + + ledger.releaseFsync() + select { + case <-handlerDone: + case <-time.After(5 * time.Second): + t.Fatal("handler did not finish after fsync release") + } + if code := w.code.Load(); code != http.StatusOK { + t.Errorf("status = %d, want 200", code) + } + if !strings.Contains(w.header.Get("Content-Type"), "application/json") { + t.Errorf("content-type = %q, want application/json", w.header.Get("Content-Type")) + } +} + +// TestAdmission_PersistenceFailureRefused (C02 3e): when the durable +// append fails, the delivery is NOT acked — 503 + Retry-After, nothing +// admitted, and the dedup entry is rolled back so the provider's retry of +// the same signed body goes through admission again. +func TestAdmission_PersistenceFailureRefused(t *testing.T) { + run := newCountingRun() + ledger := &memLedger{fail: errors.New("disk full")} + queue := newAdmissionQueue(ledger, run.run, func(string, ...any) {}) + dedup := autodeploy.NewDeliveryDedup() + handler := newWebhookHandler(webhookHandlerConfig{ + secret: "s3cret", + branch: "main", + app: "myapp", + dedup: dedup, + ledger: ledger, + queue: queue, + logf: func(string, ...any) {}, + }) + + rec := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"abc"}`) + + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("status = %d, want 503 (never ack what isn't durable)", rec.Code) + } + if rec.Header().Get("Retry-After") == "" { + t.Error("503 must carry Retry-After so the provider retries") + } + workerLive, pending := queue.snapshot() + if workerLive || pending != nil { + t.Errorf("queue admitted work despite persistence failure (worker=%v pending=%v)", workerLive, pending != nil) + } + if got := len(ledger.snapshot()); got != 0 { + t.Errorf("ledger records = %d, want 0", got) + } + + // The dedup rollback is what makes the provider's retry work: the + // same signed body must now be admitted normally once persistence + // recovers. + ledger.mu.Lock() + ledger.fail = nil + ledger.mu.Unlock() + rec2 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"abc"}`) + if rec2.Code != http.StatusOK { + t.Errorf("retry after persistence failure: status = %d, want 200 admitted", rec2.Code) + } + run.waitCall(t) + run.waitIdle(t, queue) + if run.count() != 1 { + t.Errorf("deploy ran %d times after recovery, want 1", run.count()) + } +} + +// TestAdmission_Supersede (C02 3c): with a deploy running and delivery A +// queued, delivery B supersedes A — A is marked superseded in the ledger, +// B takes the slot, B's response says so, and when the running deploy +// finishes only B's trigger runs. +func TestAdmission_Supersede(t *testing.T) { + block := make(chan struct{}) + run := newCountingRun().blocking(block) + handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) + + // W starts running (blocks in the deploy). + if rec := postSigned(t, handler, "s3cret", "w", `{"ref":"refs/heads/main","after":"w"}`); rec.Code != http.StatusOK { + t.Fatalf("running delivery status = %d", rec.Code) + } else if !strings.Contains(rec.Body.String(), `"running"`) { + t.Errorf("first delivery disposition = %q, want running", rec.Body.String()) + } + run.waitCall(t) // W is inside the deploy now + + // A queues behind it. + if rec := postSigned(t, handler, "s3cret", "a", `{"ref":"refs/heads/main","after":"a"}`); rec.Code != http.StatusOK { + t.Fatalf("queued delivery status = %d", rec.Code) + } else if !strings.Contains(rec.Body.String(), `"queued"`) { + t.Errorf("second delivery disposition = %q, want queued", rec.Body.String()) + } + + // B supersedes A. + if rec := postSigned(t, handler, "s3cret", "b", `{"ref":"refs/heads/main","after":"b"}`); rec.Code != http.StatusOK { + t.Fatalf("superseding delivery status = %d", rec.Code) + } else if !strings.Contains(rec.Body.String(), `"superseded"`) { + t.Errorf("third delivery disposition = %q, want superseded", rec.Body.String()) + } + + // A is marked superseded in the ledger, by B. + superseded := ledger.byKind(autodeploy.AdmissionKindSuperseded) + if len(superseded) != 1 { + t.Fatalf("superseded records = %d, want 1 (A)", len(superseded)) + } + if superseded[0].Delivery != "a" { + t.Errorf("superseded record names delivery %q, want a", superseded[0].Delivery) + } + bAdmitted := ledger.byKind(autodeploy.AdmissionKindAdmitted) + var bRec *autodeploy.AdmissionRecord + for i := range bAdmitted { + if bAdmitted[i].Delivery == "b" { + bRec = &bAdmitted[i] + } + } + if bRec == nil || superseded[0].SupersededBy != bRec.ID { + t.Errorf("A's superseded_by = %q, want B's admission id %v", superseded[0].SupersededBy, bRec) + } + + // Let the running deploy finish: only B runs next (never A). + close(block) + run.waitCall(t) // B + run.waitIdle(t, queue) + if run.count() != 2 { + t.Errorf("deploy ran %d times (W then B only), want 2", run.count()) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindProcessed)); got != 2 { + t.Errorf("processed records = %d, want 2 (W and B)", got) + } +} + +// TestAdmission_NoGoroutinePileup (C02 3d, the red test that failed +// against the fire-and-forget handler): 10 rapid deliveries during one +// long running deploy collapse to one running + one newest pending. No +// per-delivery spawn: after the running deploy finishes, exactly one more +// deploy (the newest) runs. +func TestAdmission_NoGoroutinePileup(t *testing.T) { + block := make(chan struct{}) + run := newCountingRun().blocking(block) + handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) + + running, queued, superseded := 0, 0, 0 + for i := 0; i < 10; i++ { + body := `{"ref":"refs/heads/main","after":"c` + string(rune('0'+i)) + `"}` + rec := postSigned(t, handler, "s3cret", "", body) + if rec.Code != http.StatusOK { + t.Fatalf("delivery %d status = %d, want 200", i, rec.Code) + } + switch { + case strings.Contains(rec.Body.String(), `"running"`): + running++ + case strings.Contains(rec.Body.String(), `"queued"`): + queued++ + case strings.Contains(rec.Body.String(), `"superseded"`): + superseded++ + } + } + + if running != 1 || queued != 1 || superseded != 8 { + t.Errorf("dispositions running=%d queued=%d superseded=%d, want 1/1/8", running, queued, superseded) + } + // Exactly one deploy is actually running; the queue holds exactly one. + run.waitCall(t) + if run.count() != 1 { + t.Errorf("deploy invocations while busy = %d, want 1", run.count()) + } + workerLive, pending := queue.snapshot() + if !workerLive || pending == nil { + t.Errorf("queue state worker=%v pending=%v, want running with one pending", workerLive, pending != nil) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindSuperseded)); got != 8 { + t.Errorf("superseded ledger records = %d, want 8", got) + } + + close(block) + run.waitCall(t) // the newest pending + run.waitIdle(t, queue) + if run.count() != 2 { + t.Errorf("total deploy invocations = %d, want 2 (first + newest)", run.count()) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindProcessed)); got != 2 { + t.Errorf("processed records = %d, want 2", got) + } +} + +// TestAdmission_ResumeAfterCrash (C02 3b): admitted-but-never-processed +// ledger entries trigger exactly one deploy on resume; processed entries +// never re-trigger; a duplicate delivery id never re-triggers; and the +// resume reseeds the dedup set from the ledger so redeliveries read as +// replays. +func TestAdmission_ResumeAfterCrash(t *testing.T) { + dir := t.TempDir() + ledgerPath := filepath.Join(dir, ".autodeploy-ledger.jsonl") + fileLedger, err := autodeploy.OpenLedger(ledgerPath) + if err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Add(-time.Minute) + mustAdmit := func(id, delivery, digest string, received time.Time) autodeploy.AdmissionRecord { + rec := autodeploy.AdmissionRecord{ + Kind: autodeploy.AdmissionKindAdmitted, ID: id, Delivery: delivery, + Digest: digest, App: "myapp", Branch: "main", Received: received, + } + if err := fileLedger.Append(rec); err != nil { + t.Fatal(err) + } + return rec + } + + // A: admitted and processed — must never re-trigger. + a := mustAdmit("id-a", "del-a", "aaaa", base) + if err := fileLedger.Append(autodeploy.AdmissionRecord{Kind: autodeploy.AdmissionKindProcessed, ID: a.ID, Digest: a.Digest, App: "myapp", Received: a.Received, At: base.Add(time.Second)}); err != nil { + t.Fatal(err) + } + // B: admitted, never processed — the crash window. + b := mustAdmit("id-b", "del-b", "bbbb", base.Add(2*time.Second)) + _ = b + + run := newCountingRun() + resumeLedger := &memLedger{} + dedup := autodeploy.NewDeliveryDedup() + queue := newAdmissionQueue(resumeLedger, run.run, func(string, ...any) {}) + if err := resumeAdmissions(ledgerPath, "myapp", resumeLedger, queue, dedup, func(string, ...any) {}); err != nil { + t.Fatal(err) + } + + run.waitCall(t) + run.waitIdle(t, queue) + if run.count() != 1 { + t.Errorf("resume ran %d deploys, want exactly 1 (the admitted-unprocessed entry)", run.count()) + } + // The resumed admission is marked processed in the resume ledger. + if got := len(resumeLedger.byKind(autodeploy.AdmissionKindProcessed)); got != 1 { + t.Errorf("resume processed marks = %d, want 1", got) + } + // Dedup reseeded from the ledger: redelivering B's digest reads as replay. + if !dedup.SeenAndRecord(autodeploy.ContentIDFromDigest("bbbb")) { + t.Error("resume did not reseed dedup with the admitted digest (redelivery would re-admit)") + } + if dedup.SeenAndRecord(autodeploy.ContentIDFromDigest("zzzz")) { + t.Error("unrelated digest incorrectly seeded into dedup") + } +} + +// TestAdmission_ResumeNewestWinsAndDupesSkipped: several pending +// admissions (including a shared delivery id and a shared digest) collapse +// to ONE deploy of the newest entry. +func TestAdmission_ResumeNewestWinsAndDupesSkipped(t *testing.T) { + dir := t.TempDir() + ledgerPath := filepath.Join(dir, ".autodeploy-ledger.jsonl") + fileLedger, err := autodeploy.OpenLedger(ledgerPath) + if err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Add(-time.Minute) + admits := []struct { + id, delivery, digest string + offset time.Duration + }{ + {"id-old", "same-delivery", "old-digest", 0}, + {"id-mid", "same-delivery", "mid-digest", time.Second}, // reused delivery id + {"id-new", "same-delivery", "new-digest", 2 * time.Second}, + } + for _, a := range admits { + if err := fileLedger.Append(autodeploy.AdmissionRecord{ + Kind: autodeploy.AdmissionKindAdmitted, ID: a.id, Delivery: a.delivery, + Digest: a.digest, App: "myapp", Branch: "main", Received: base.Add(a.offset), + }); err != nil { + t.Fatal(err) + } + } + + run := newCountingRun() + resumeLedger := &memLedger{} + queue := newAdmissionQueue(resumeLedger, run.run, func(string, ...any) {}) + if err := resumeAdmissions(ledgerPath, "myapp", resumeLedger, queue, autodeploy.NewDeliveryDedup(), func(string, ...any) {}); err != nil { + t.Fatal(err) + } + + run.waitCall(t) + run.waitIdle(t, queue) + if run.count() != 1 { + t.Errorf("resume ran %d deploys, want 1 (newest wins; duplicates skipped)", run.count()) + } + // The older pendings were marked superseded by the newest. + superseded := resumeLedger.byKind(autodeploy.AdmissionKindSuperseded) + if len(superseded) != 2 { + t.Fatalf("resume superseded marks = %d, want 2", len(superseded)) + } + for _, s := range superseded { + if s.SupersededBy != "id-new" { + t.Errorf("superseded_by = %q, want id-new", s.SupersededBy) + } + } +} + +// TestAdmission_ResumeEmptyAndMissing: no ledger file and an all-processed +// ledger are both quiet resumes. +func TestAdmission_ResumeEmptyAndMissing(t *testing.T) { + run := newCountingRun() + q := newAdmissionQueue(&memLedger{}, run.run, func(string, ...any) {}) + + if err := resumeAdmissions(filepath.Join(t.TempDir(), "nope.jsonl"), "myapp", &memLedger{}, q, autodeploy.NewDeliveryDedup(), func(string, ...any) {}); err != nil { + t.Fatalf("missing ledger must not error: %v", err) + } + + dir := t.TempDir() + path := filepath.Join(dir, "ledger.jsonl") + fl, err := autodeploy.OpenLedger(path) + if err != nil { + t.Fatal(err) + } + if err := fl.Append(autodeploy.AdmissionRecord{Kind: autodeploy.AdmissionKindAdmitted, ID: "x", Digest: "d", App: "myapp", Received: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + if err := fl.Append(autodeploy.AdmissionRecord{Kind: autodeploy.AdmissionKindProcessed, ID: "x", Digest: "d", App: "myapp", Received: time.Now().UTC(), At: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + if err := resumeAdmissions(path, "myapp", &memLedger{}, q, autodeploy.NewDeliveryDedup(), func(string, ...any) {}); err != nil { + t.Fatal(err) + } + run.waitIdle(t, q) + if run.count() != 0 { + t.Errorf("all-processed ledger re-triggered %d deploys, want 0", run.count()) + } +} + +// TestAdmission_SameDigestWhileQueuedIsDuplicate: a redelivery of the +// exact same signed body while it sits in the pending slot is a duplicate +// (dedup path), not a second admission. +func TestAdmission_SameDigestWhileQueuedIsDuplicate(t *testing.T) { + block := make(chan struct{}) + run := newCountingRun().blocking(block) + handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) + + postSigned(t, handler, "s3cret", "w", `{"ref":"refs/heads/main","after":"w"}`) + run.waitCall(t) + body := `{"ref":"refs/heads/main","after":"q"}` + if rec := postSigned(t, handler, "s3cret", "q1", body); !strings.Contains(rec.Body.String(), `"queued"`) { + t.Fatalf("first copy disposition = %q, want queued", rec.Body.String()) + } + if rec := postSigned(t, handler, "s3cret", "q2", body); !strings.Contains(rec.Body.String(), `"duplicate"`) { + t.Fatalf("redelivery while pending = %q, want duplicate", rec.Body.String()) + } + + close(block) + run.waitCall(t) + run.waitIdle(t, queue) + if run.count() != 2 { + t.Errorf("deploy ran %d times, want 2", run.count()) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindAdmitted)); got != 2 { + t.Errorf("admissions = %d, want 2 (W and one Q — the redelivery is not admitted)", got) + } +} + +// TestAdmission_LedgerFileDurability: the real FileLedger round-trips +// through ParseLedger and FoldAdmissions, ignores a torn tail, and refuses +// mid-file corruption. +func TestAdmission_LedgerFileDurability(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ledger.jsonl") + fl, err := autodeploy.OpenLedger(path) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + recs := []autodeploy.AdmissionRecord{ + {Kind: autodeploy.AdmissionKindAdmitted, ID: "1", Digest: "d1", App: "myapp", Received: now}, + {Kind: autodeploy.AdmissionKindSuperseded, ID: "1", Digest: "d1", App: "myapp", Received: now, SupersededBy: "2", At: now}, + {Kind: autodeploy.AdmissionKindAdmitted, ID: "2", Digest: "d2", App: "myapp", Received: now.Add(time.Second)}, + {Kind: autodeploy.AdmissionKindProcessed, ID: "2", Digest: "d2", App: "myapp", Received: now.Add(time.Second), At: now.Add(2 * time.Second)}, + } + for _, r := range recs { + if err := fl.Append(r); err != nil { + t.Fatal(err) + } + } + fl.Close() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + parsed, err := autodeploy.ParseLedger(data) + if err != nil { + t.Fatal(err) + } + if len(parsed) != 4 { + t.Fatalf("parsed %d records, want 4", len(parsed)) + } + pending, digests := autodeploy.FoldAdmissions(parsed) + if len(pending) != 0 { + t.Errorf("pending = %d, want 0 (1 superseded, 2 processed)", len(pending)) + } + if len(digests) != 2 { + t.Errorf("digests = %d, want 2", len(digests)) + } + + // Torn tail (crash mid-append, no trailing newline) is ignored. + torn := append(data, []byte(`{"kind":"admitted","id":"3","dig`)...) + parsed, err = autodeploy.ParseLedger(torn) + if err != nil { + t.Fatalf("torn tail must be ignored: %v", err) + } + if len(parsed) != 4 { + t.Errorf("torn tail parsed %d records, want 4", len(parsed)) + } + + // Mid-file corruption fails closed. + corrupt := []byte("{\"kind\":\"admitted\"\n{\"kind\":\"processed\"}\n") + if _, err := autodeploy.ParseLedger(corrupt); !errors.Is(err, autodeploy.ErrLedgerCorrupt) { + t.Errorf("mid-file corruption = %v, want ErrLedgerCorrupt", err) + } + + // Unknown kind fails closed. + unknown := []byte("{\"kind\":\"weird\"}\n") + if _, err := autodeploy.ParseLedger(unknown); !errors.Is(err, autodeploy.ErrLedgerCorrupt) { + t.Errorf("unknown kind = %v, want ErrLedgerCorrupt", err) + } + + // Ledger file is private. + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Errorf("ledger mode = %v, want 0600", info.Mode().Perm()) + } +} diff --git a/internal/cli/autodeploy_serve.go b/internal/cli/autodeploy_serve.go index 54d8e22..6560aee 100644 --- a/internal/cli/autodeploy_serve.go +++ b/internal/cli/autodeploy_serve.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -101,7 +102,7 @@ func runAutoDeployServe(app, branch string, port int, strictEnv bool) error { // Restore delivery dedup state across restarts (best-effort — losing // this on a restart just means a very recent replay could briefly slip - // through, not a hard failure). + // through, not a hard failure; the admission ledger below reseeds it). dedupPath := fmt.Sprintf("/deployments/%s/.autodeploy-dedup.json", app) dedupData, _ := os.ReadFile(dedupPath) dedup := autodeploy.LoadDeliveryDedup(dedupData) @@ -113,15 +114,47 @@ func runAutoDeployServe(app, branch string, port int, strictEnv bool) error { fmt.Fprintf(out, "%s "+format+"\n", append([]any{time.Now().UTC().Format(time.RFC3339)}, args...)...) } + // The admission ledger (C02): every 200 this process sends is backed by + // an fsynced record here, and restarts replay admitted-but-never- + // processed deliveries from it. + ledgerPath := fmt.Sprintf("/deployments/%s/.autodeploy-ledger.jsonl", app) + ledger, err := autodeploy.OpenLedger(ledgerPath) + if err != nil { + return err + } + defer ledger.Close() + + // Bounded queueing: one worker, one newest-wins pending slot. The + // deploy itself runs in the worker — never a goroutine per delivery. + queue := newAdmissionQueue(ledger, func(changedFiles []string, filesKnown bool) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + if err := triggerAutoDeploy(ctx, executor, app, branch, buildDir, out, changedFiles, filesKnown, strictEnv); err != nil { + logf("deploy failed: %v", err) + } else { + logf("deploy complete") + } + }, logf) + + if err := resumeAdmissions(ledgerPath, app, ledger, queue, dedup, logf); err != nil { + return fmt.Errorf("resuming webhook admissions: %w", err) + } + // Dedup persistence is serialized AND atomic (audit A36): two // concurrent requests used to snapshot and os.WriteFile the same file // independently, so an older snapshot could overwrite a newer one, // overlapping writes could truncate, and every error was ignored. + // It stays best-effort: the admission LEDGER is the durable record; + // a lost dedup snapshot degrades to one redundant deploy of the + // branch tip, never a lost admission. var dedupMu sync.Mutex handler := newWebhookHandler(webhookHandlerConfig{ secret: secret, branch: branch, + app: app, dedup: dedup, + ledger: ledger, + queue: queue, logf: logf, onDedupChanged: func() { dedupMu.Lock() @@ -140,22 +173,6 @@ func runAutoDeployServe(app, branch string, port int, strictEnv bool) error { logf("could not publish webhook dedup state: %v", err) } }, - trigger: func(changedFiles []string, filesKnown bool) { - // Deploy asynchronously so the webhook response isn't held - // open for a potentially multi-minute build — matches - // providers' expectation of a prompt response, without - // needing the old bash listener's `nohup ... &` detached- - // process trick. - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - if err := triggerAutoDeploy(ctx, executor, app, branch, buildDir, out, changedFiles, filesKnown, strictEnv); err != nil { - logf("deploy failed: %v", err) - } else { - logf("deploy complete") - } - }() - }, }) mux := http.NewServeMux() @@ -187,32 +204,46 @@ func runAutoDeployServe(app, branch string, port int, strictEnv bool) error { // webhookHandlerConfig holds newWebhookHandler's dependencies, injected // rather than closed over directly so the request-handling logic (HMAC -// verification, dedup, response codes) is unit-testable with -// httptest.NewRecorder without touching the filesystem or triggering a -// real deploy. +// verification, dedup, admission durability, response codes) is +// unit-testable with httptest without touching the filesystem or +// triggering a real deploy. type webhookHandlerConfig struct { secret string // branch is the ref this listener watches; only push events for it may // trigger a deploy (audit F40). Empty accepts any push (tests). branch string - dedup *autodeploy.DeliveryDedup - logf func(format string, args ...any) - // onDedupChanged is called after a new (non-replayed) delivery ID is - // recorded, so the caller can persist the dedup snapshot. Optional. + // app names the admissions written to the ledger (one serve process + // per app). + app string + dedup *autodeploy.DeliveryDedup + logf func(format string, args ...any) + // ledger is the durable admission record (C02): the handler acks 200 + // only after the fsynced append of the admission succeeds. + ledger autodeploy.LedgerAppender + // queue holds the bounded one-running-plus-one-pending deploy slot. + queue *admissionQueue + // onDedupChanged is called after a delivery is DURABLY admitted, so + // the caller can persist the dedup snapshot. Best-effort. Optional. onDedupChanged func() - // trigger is called exactly once per accepted, non-replayed webhook — - // the actual deploy kickoff. Never called for a rejected or replayed - // request. It receives the files the push touched and whether that set - // is reliable (see autodeploy.ChangedFiles); the deploy step uses them - // for monorepo path filtering. - trigger func(changedFiles []string, filesKnown bool) } // newWebhookHandler returns the HTTP handler for the webhook endpoint: // verifies the request (GitHub HMAC or GitLab token, whichever header is -// present), rejects invalid or replayed deliveries, and calls cfg.trigger -// exactly once for anything else. +// present), rejects invalid or replayed deliveries, durably admits pushes +// to the watched branch (200 only after the admission record is fsynced — +// C02), and enqueues them on the bounded queue. A persistence failure is +// a 503 + Retry-After: never ack what isn't durable. func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { + writeJSON := func(w http.ResponseWriter, code int, v any) { + body, err := json.Marshal(v) + if err != nil { + w.WriteHeader(code) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + w.Write(body) + } return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) @@ -267,17 +298,15 @@ func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { // ID carrying different authenticated content used to suppress a // distinct event (audit A36). contentSum := sha256.Sum256(body) - contentID := "content:" + hex.EncodeToString(contentSum[:]) + digest := hex.EncodeToString(contentSum[:]) + contentID := autodeploy.ContentIDFromDigest(digest) if cfg.dedup.SeenAndRecord(contentID) { - w.WriteHeader(http.StatusOK) + writeJSON(w, http.StatusOK, admissionResponse{Status: "duplicate"}) if cfg.logf != nil { cfg.logf("ignored replayed webhook content") } return } - if cfg.onDedupChanged != nil { - cfg.onDedupChanged() - } // Bind the deploy to THIS event: only a push to the watched branch // may trigger it; pings, tags, other branches, and branch deletions @@ -290,22 +319,240 @@ func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { return } - w.WriteHeader(http.StatusOK) + // Parse the changed-file set from the push body for monorepo path + // filtering. filesKnown=false (unknown provider, truncated or + // tag/ping payload) means the deploy step must not skip. + changedFiles, filesKnown := autodeploy.ChangedFiles(body) + + // DURABLE ADMISSION (C02): the record is fsynced BEFORE the 200. + // A crash immediately after the response still leaves the admitted + // delivery discoverable by restart resume. + rec := autodeploy.AdmissionRecord{ + Kind: autodeploy.AdmissionKindAdmitted, + ID: autodeploy.NewAdmissionID(), + Delivery: deliveryID, + Digest: digest, + App: cfg.app, + Branch: cfg.branch, + Received: time.Now().UTC(), + } + if err := cfg.ledger.Append(rec); err != nil { + // Not durable → never ack. Un-record the dedup entry so the + // provider's retry of the SAME signed body goes through + // admission again instead of being swallowed as a replay of + // something that was never admitted. + cfg.dedup.Unrecord(contentID) + if cfg.logf != nil { + cfg.logf("admission not durable, rejecting (provider should retry): %v", err) + } + w.Header().Set("Retry-After", "5") + writeJSON(w, http.StatusServiceUnavailable, admissionResponse{Status: "error", Error: "admission could not be made durable"}) + return + } + if cfg.onDedupChanged != nil { + cfg.onDedupChanged() + } + + disposition := cfg.queue.admit(rec, changedFiles, filesKnown) + writeJSON(w, http.StatusOK, admissionResponse{Status: "admitted", Disposition: string(disposition)}) if cfg.logf != nil { if deliveryID != "" { - cfg.logf("accepted webhook (delivery %s), triggering deploy", deliveryID) + cfg.logf("accepted webhook (delivery %s), admission %s %s", deliveryID, rec.ID, disposition) } else { - cfg.logf("accepted webhook, triggering deploy") + cfg.logf("accepted webhook, admission %s %s", rec.ID, disposition) } } - if cfg.trigger != nil { - // Parse the changed-file set from the push body for monorepo - // path filtering. filesKnown=false (unknown provider, truncated - // or tag/ping payload) means the deploy step must not skip. - changedFiles, filesKnown := autodeploy.ChangedFiles(body) - cfg.trigger(changedFiles, filesKnown) + } +} + +// admissionResponse is the small JSON body on admission replies so the +// provider (and tests) can tell what happened: running (deploy starting +// now), queued (one deploy running, this one is the pending newest), +// superseded (replaced an older pending delivery), duplicate (replay). +type admissionResponse struct { + Status string `json:"status"` + Disposition string `json:"disposition,omitempty"` + Error string `json:"error,omitempty"` +} + +// admissionDisposition reports what the bounded queue did with a durably +// admitted delivery. +type admissionDisposition string + +const ( + dispositionRunning admissionDisposition = "running" + dispositionQueued admissionDisposition = "queued" + dispositionSuperseded admissionDisposition = "superseded" +) + +// queuedAdmission is PENDING WORK AS A RECORD, never a blocked goroutine: +// the single worker picks it up when the running deploy finishes. +type queuedAdmission struct { + rec autodeploy.AdmissionRecord + changedFiles []string + filesKnown bool +} + +// admissionQueue is the bounded webhook deploy queue (C02): at most ONE +// running deploy plus ONE pending slot per app, newest-wins. A delivery +// arriving while both are busy SUPERSEDES the queued one (the running +// deploy is never cancelled mid-flight — cancellation propagation is +// deliberately out of scope; the newest deploy runs next instead). +type admissionQueue struct { + mu sync.Mutex + workerLive bool + pending *queuedAdmission + + ledger autodeploy.LedgerAppender + run func(changedFiles []string, filesKnown bool) + logf func(format string, args ...any) +} + +func newAdmissionQueue(ledger autodeploy.LedgerAppender, run func(changedFiles []string, filesKnown bool), logf func(format string, args ...any)) *admissionQueue { + return &admissionQueue{ledger: ledger, run: run, logf: logf} +} + +// admit places a durably admitted delivery on the queue and reports the +// disposition. Called from the request goroutine after the ledger append. +func (q *admissionQueue) admit(rec autodeploy.AdmissionRecord, changedFiles []string, filesKnown bool) admissionDisposition { + q.mu.Lock() + defer q.mu.Unlock() + item := &queuedAdmission{rec: rec, changedFiles: changedFiles, filesKnown: filesKnown} + switch { + case q.workerLive && q.pending != nil: + old := q.pending + q.pending = item + q.markSupersededLocked(old.rec, rec.ID) + return dispositionSuperseded + case q.workerLive: + q.pending = item + return dispositionQueued + default: + q.pending = item + q.workerLive = true + go q.worker() + return dispositionRunning + } +} + +// worker is the ONLY deploy runner: one goroutine at a time, draining the +// pending slot. It exits when the queue is empty; the next admit restarts +// it — so rapid deliveries during a long deploy never spawn per-delivery +// goroutines. +func (q *admissionQueue) worker() { + for { + q.mu.Lock() + item := q.pending + if item == nil { + q.workerLive = false + q.mu.Unlock() + return + } + q.pending = nil + q.mu.Unlock() + + if q.run != nil { + q.run(item.changedFiles, item.filesKnown) } + q.markProcessed(item.rec) + } +} + +// markSupersededLocked records the newest-wins replacement in the ledger. +// Best-effort: a failed mark leaves both records pending, and resume's +// newest-per-app rule still picks the newer one — the event is not lost. +func (q *admissionQueue) markSupersededLocked(old autodeploy.AdmissionRecord, byID string) { + err := q.ledger.Append(autodeploy.AdmissionRecord{ + Kind: autodeploy.AdmissionKindSuperseded, + ID: old.ID, + Delivery: old.Delivery, + Digest: old.Digest, + App: old.App, + Branch: old.Branch, + Received: old.Received, + SupersededBy: byID, + At: time.Now().UTC(), + }) + if err != nil && q.logf != nil { + q.logf("could not mark admission %s superseded: %v (resume still picks the newest)", old.ID, err) + } +} + +// markProcessed records completion in the ledger so restart resume never +// re-triggers a finished deploy. Best-effort: a failed mark can replay ONE +// deploy of the branch tip on resume — idempotent at the engine (fetch + +// same-version redeploy), never a lost event. +func (q *admissionQueue) markProcessed(rec autodeploy.AdmissionRecord) { + err := q.ledger.Append(autodeploy.AdmissionRecord{ + Kind: autodeploy.AdmissionKindProcessed, + ID: rec.ID, + Delivery: rec.Delivery, + Digest: rec.Digest, + App: rec.App, + Branch: rec.Branch, + Received: rec.Received, + At: time.Now().UTC(), + }) + if err != nil && q.logf != nil { + q.logf("could not mark admission %s processed: %v (resume may replay this deploy once)", rec.ID, err) + } +} + +// snapshot exposes the bounded state for tests and diagnostics. +func (q *admissionQueue) snapshot() (workerLive bool, pending *queuedAdmission) { + q.mu.Lock() + defer q.mu.Unlock() + return q.workerLive, q.pending +} + +// resumeAdmissions replays admitted-but-never-processed deliveries from +// the ledger after a restart (C02): newest per app wins, older pendings +// are marked superseded, and the recent admitted digests reseed the replay +// dedup (the dedup file is best-effort). This is the correct webhook +// contract — the provider will not redeliver an event it was told was +// accepted, so replaying admitted-not-processed work IS the job (unlike a +// UI admission write, where replay is the bug). +func resumeAdmissions(ledgerPath, app string, ledger autodeploy.LedgerAppender, q *admissionQueue, dedup *autodeploy.DeliveryDedup, logf func(format string, args ...any)) error { + data, err := os.ReadFile(ledgerPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("reading webhook admission ledger: %w", err) + } + recs, err := autodeploy.ParseLedger(data) + if err != nil { + return err + } + pending, digests := autodeploy.FoldAdmissions(recs) + autodeploy.SeedDedupFromLedger(dedup, digests, time.Now().UTC()) + newest := autodeploy.NewestPending(pending, app) + if newest == nil { + return nil + } + for _, p := range pending { + if p.App != app || p.ID == newest.ID { + continue + } + if err := ledger.Append(autodeploy.AdmissionRecord{ + Kind: autodeploy.AdmissionKindSuperseded, + ID: p.ID, + Delivery: p.Delivery, + Digest: p.Digest, + App: p.App, + Branch: p.Branch, + Received: p.Received, + SupersededBy: newest.ID, + At: time.Now().UTC(), + }); err != nil && logf != nil { + logf("could not mark stale admission %s superseded during resume: %v", p.ID, err) + } + } + if logf != nil { + logf("resuming admitted-but-unprocessed webhook delivery %s (received %s; changed-file list unrecoverable post-crash — deploying fail-open)", newest.ID, newest.Received.Format(time.RFC3339)) } + q.admit(*newest, nil, false) + return nil } // triggerAutoDeploy fetches the watched branch, builds, and deploys — diff --git a/internal/cli/autodeploy_serve_test.go b/internal/cli/autodeploy_serve_test.go index 0253be6..1e424e4 100644 --- a/internal/cli/autodeploy_serve_test.go +++ b/internal/cli/autodeploy_serve_test.go @@ -7,7 +7,9 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" + "time" "github.com/useteploy/teploy/internal/autodeploy" "github.com/useteploy/teploy/internal/config" @@ -19,45 +21,162 @@ func githubSign(secret string, body []byte) string { return "sha256=" + hex.EncodeToString(mac.Sum(nil)) } -// TestWebhookHandler_ValidSignatureTriggersDeployOnce is the direct -// regression test for the old autodeploy's worst bug: the generated bash -// listener verified the signature correctly but never called anything -// resembling a real deploy — it only built an image and stopped. This -// confirms a valid, well-formed webhook actually calls trigger exactly -// once. -func TestWebhookHandler_ValidSignatureTriggersDeployOnce(t *testing.T) { - secret := "s3cret" - body := []byte(`{"ref":"refs/heads/main"}`) - triggerCount := 0 +// memLedger is an in-memory LedgerAppender for handler tests: records every +// append, optionally fails. +type memLedger struct { + mu sync.Mutex + records []autodeploy.AdmissionRecord + fail error +} + +func (m *memLedger) Append(rec autodeploy.AdmissionRecord) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return m.fail + } + m.records = append(m.records, rec) + return nil +} + +func (m *memLedger) snapshot() []autodeploy.AdmissionRecord { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]autodeploy.AdmissionRecord, len(m.records)) + copy(out, m.records) + return out +} +func (m *memLedger) byKind(kind string) []autodeploy.AdmissionRecord { + var out []autodeploy.AdmissionRecord + for _, rec := range m.snapshot() { + if rec.Kind == kind { + out = append(out, rec) + } + } + return out +} + +// countingRun records deploy invocations. Each entry signals started +// (the deploy was entered — even if it then blocks) and each completion +// signals done. +type countingRun struct { + mu sync.Mutex + calls []time.Time + started chan struct{} + done chan struct{} + block chan struct{} // non-nil: each call blocks until closed +} + +func newCountingRun() *countingRun { + return &countingRun{started: make(chan struct{}, 64), done: make(chan struct{}, 64)} +} + +func (c *countingRun) blocking(block chan struct{}) *countingRun { + c.block = block + return c +} + +func (c *countingRun) run(_ []string, _ bool) { + c.mu.Lock() + c.calls = append(c.calls, time.Now()) + block := c.block + c.mu.Unlock() + c.started <- struct{}{} + if block != nil { + <-block + } + c.done <- struct{}{} +} + +func (c *countingRun) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.calls) +} + +func (c *countingRun) waitCall(t *testing.T) { + t.Helper() + select { + case <-c.started: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for a deploy invocation") + } +} + +func (c *countingRun) waitIdle(t *testing.T, q *admissionQueue) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + workerLive, pending := q.snapshot() + if !workerLive && pending == nil { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatal("queue did not drain to idle within timeout") +} + +// newAdmissionStack wires a handler + queue + ledger with an injectable +// deploy runner, the shape runAutoDeployServe uses. +func newAdmissionStack(secret, branch, app string, run func([]string, bool)) (http.HandlerFunc, *memLedger, *admissionQueue) { + ledger := &memLedger{} + queue := newAdmissionQueue(ledger, run, func(string, ...any) {}) handler := newWebhookHandler(webhookHandlerConfig{ - secret: secret, - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { triggerCount++ }, + secret: secret, + branch: branch, + app: app, + dedup: autodeploy.NewDeliveryDedup(), + ledger: ledger, + queue: queue, + logf: func(string, ...any) {}, }) + return handler, ledger, queue +} - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(body))) - req.Header.Set("X-Hub-Signature-256", githubSign(secret, body)) - req.Header.Set("X-GitHub-Delivery", "delivery-1") +func postSigned(t *testing.T, handler http.HandlerFunc, secret, delivery, body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("X-Hub-Signature-256", githubSign(secret, []byte(body))) + if delivery != "" { + req.Header.Set("X-GitHub-Delivery", delivery) + } rec := httptest.NewRecorder() - handler(rec, req) + return rec +} + +// TestWebhookHandler_ValidSignatureTriggersDeployOnce is the direct +// regression test for the old autodeploy's worst bug: the generated bash +// listener verified the signature correctly but never called anything +// resembling a real deploy — it only built an image and stopped. This +// confirms a valid, well-formed webhook is durably admitted and runs the +// deploy exactly once. +func TestWebhookHandler_ValidSignatureTriggersDeployOnce(t *testing.T) { + run := newCountingRun() + handler, ledger, queue := newAdmissionStack("s3cret", "", "myapp", run.run) + + rec := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) if rec.Code != http.StatusOK { t.Errorf("status = %d, want 200", rec.Code) } - if triggerCount != 1 { - t.Errorf("trigger called %d times, want 1", triggerCount) + run.waitCall(t) + run.waitIdle(t, queue) + if run.count() != 1 { + t.Errorf("deploy ran %d times, want 1", run.count()) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindAdmitted)); got != 1 { + t.Errorf("admitted records = %d, want 1", got) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindProcessed)); got != 1 { + t.Errorf("processed records = %d, want 1", got) } } func TestWebhookHandler_InvalidSignatureNeverTriggers(t *testing.T) { - triggered := false - handler := newWebhookHandler(webhookHandlerConfig{ - secret: "s3cret", - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { triggered = true }, - }) + run := newCountingRun() + handler, _, _ := newAdmissionStack("s3cret", "", "myapp", run.run) req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"ref":"refs/heads/main"}`)) req.Header.Set("X-Hub-Signature-256", "sha256=0000000000000000000000000000000000000000000000000000000000000000") @@ -68,18 +187,14 @@ func TestWebhookHandler_InvalidSignatureNeverTriggers(t *testing.T) { if rec.Code != http.StatusUnauthorized { t.Errorf("status = %d, want 401", rec.Code) } - if triggered { + if run.count() != 0 { t.Error("an invalid signature must never trigger a deploy") } } func TestWebhookHandler_NoSignatureNeverTriggers(t *testing.T) { - triggered := false - handler := newWebhookHandler(webhookHandlerConfig{ - secret: "s3cret", - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { triggered = true }, - }) + run := newCountingRun() + handler, _, _ := newAdmissionStack("s3cret", "", "myapp", run.run) req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{}`)) rec := httptest.NewRecorder() @@ -89,7 +204,7 @@ func TestWebhookHandler_NoSignatureNeverTriggers(t *testing.T) { if rec.Code != http.StatusUnauthorized { t.Errorf("status = %d, want 401", rec.Code) } - if triggered { + if run.count() != 0 { t.Error("a request with no signature header must never trigger a deploy") } } @@ -99,67 +214,52 @@ func TestWebhookHandler_NoSignatureNeverTriggers(t *testing.T) { // valid payload+signature could be replayed indefinitely to re-trigger // deploys. func TestWebhookHandler_ReplayedDeliveryIgnored(t *testing.T) { - secret := "s3cret" - body := []byte(`{"ref":"refs/heads/main"}`) - triggerCount := 0 - dedup := autodeploy.NewDeliveryDedup() + run := newCountingRun() + handler, _, queue := newAdmissionStack("s3cret", "", "myapp", run.run) - handler := newWebhookHandler(webhookHandlerConfig{ - secret: secret, - dedup: dedup, - trigger: func(_ []string, _ bool) { triggerCount++ }, - }) + rec1 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) + rec2 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) - makeReq := func() *http.Request { - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(body))) - req.Header.Set("X-Hub-Signature-256", githubSign(secret, body)) - req.Header.Set("X-GitHub-Delivery", "delivery-1") - return req - } - - handler(httptest.NewRecorder(), makeReq()) - rec2 := httptest.NewRecorder() - handler(rec2, makeReq()) - - if triggerCount != 1 { - t.Errorf("trigger called %d times across a replayed delivery, want 1", triggerCount) + run.waitCall(t) + run.waitIdle(t, queue) + if run.count() != 1 { + t.Errorf("deploy ran %d times across a replayed delivery, want 1", run.count()) } // A replay is a no-op, not a rejection — provider shouldn't retry harder. if rec2.Code != http.StatusOK { t.Errorf("replayed delivery status = %d, want 200 (no-op, not a failure)", rec2.Code) } + if !strings.Contains(rec2.Body.String(), `"duplicate"`) { + t.Errorf("replayed delivery body = %q, want duplicate status", rec2.Body.String()) + } + if rec1.Code != http.StatusOK { + t.Errorf("first delivery status = %d, want 200", rec1.Code) + } } func TestWebhookHandler_GitLabToken(t *testing.T) { - secret := "s3cret" - triggerCount := 0 - handler := newWebhookHandler(webhookHandlerConfig{ - secret: secret, - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { triggerCount++ }, - }) + run := newCountingRun() + handler, _, queue := newAdmissionStack("s3cret", "", "myapp", run.run) req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"ref":"refs/heads/main"}`)) - req.Header.Set("X-Gitlab-Token", secret) + req.Header.Set("X-Gitlab-Token", "s3cret") rec := httptest.NewRecorder() handler(rec, req) + run.waitCall(t) + run.waitIdle(t, queue) if rec.Code != http.StatusOK { t.Errorf("status = %d, want 200", rec.Code) } - if triggerCount != 1 { - t.Errorf("trigger called %d times, want 1", triggerCount) + if run.count() != 1 { + t.Errorf("deploy ran %d times, want 1", run.count()) } } func TestWebhookHandler_GitLabWrongToken(t *testing.T) { - triggered := false - handler := newWebhookHandler(webhookHandlerConfig{ - secret: "s3cret", - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { triggered = true }, - }) + run := newCountingRun() + handler, _, _ := newAdmissionStack("s3cret", "", "myapp", run.run) req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{}`)) req.Header.Set("X-Gitlab-Token", "wrong") @@ -170,18 +270,14 @@ func TestWebhookHandler_GitLabWrongToken(t *testing.T) { if rec.Code != http.StatusUnauthorized { t.Errorf("status = %d, want 401", rec.Code) } - if triggered { + if run.count() != 0 { t.Error("wrong GitLab token must never trigger a deploy") } } func TestWebhookHandler_RejectsNonPOST(t *testing.T) { - triggered := false - handler := newWebhookHandler(webhookHandlerConfig{ - secret: "s3cret", - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { triggered = true }, - }) + run := newCountingRun() + handler, _, _ := newAdmissionStack("s3cret", "", "myapp", run.run) req := httptest.NewRequest(http.MethodGet, "/", nil) rec := httptest.NewRecorder() @@ -191,31 +287,48 @@ func TestWebhookHandler_RejectsNonPOST(t *testing.T) { if rec.Code != http.StatusMethodNotAllowed { t.Errorf("status = %d, want 405", rec.Code) } - if triggered { + if run.count() != 0 { t.Error("a GET request must never trigger a deploy") } } -func TestWebhookHandler_OnDedupChangedCalledOnNewDelivery(t *testing.T) { - secret := "s3cret" - body := []byte(`{}`) +// TestWebhookHandler_OnDedupChangedCalledAfterDurableAdmission pins the +// C02 ordering: the dedup snapshot persist fires only after the admission +// is durable, so a dedup entry on disk always corresponds to a ledger +// admission. Non-push events (pings, other branches) are acknowledged +// without persisting dedup — their replay after a restart is a harmless +// no-op ack, never a lost deploy. +func TestWebhookHandler_OnDedupChangedCalledAfterDurableAdmission(t *testing.T) { + run := newCountingRun() + ledger := &memLedger{} + queue := newAdmissionQueue(ledger, run.run, func(string, ...any) {}) changedCount := 0 - handler := newWebhookHandler(webhookHandlerConfig{ - secret: secret, + secret: "s3cret", + branch: "main", + app: "myapp", dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) {}, + ledger: ledger, + queue: queue, + logf: func(string, ...any) {}, onDedupChanged: func() { changedCount++ }, }) - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(body))) - req.Header.Set("X-Hub-Signature-256", githubSign(secret, body)) - req.Header.Set("X-GitHub-Delivery", "delivery-1") - - handler(httptest.NewRecorder(), req) + postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"a"}`) if changedCount != 1 { - t.Errorf("onDedupChanged called %d times, want 1", changedCount) + t.Errorf("onDedupChanged called %d times after durable admission, want 1", changedCount) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindAdmitted)); got != 1 { + t.Errorf("admitted = %d before onDedupChanged fired, want 1 (persist follows durability)", got) } + + // A ping is acknowledged but never persisted to dedup (no admission). + changedCount = 0 + postSigned(t, handler, "s3cret", "delivery-2", `{}`) + if changedCount != 0 { + t.Errorf("onDedupChanged fired %d times for a ping, want 0", changedCount) + } + run.waitIdle(t, queue) } // audit F40: only a push to the WATCHED branch may trigger a deploy. A ping, @@ -223,7 +336,6 @@ func TestWebhookHandler_OnDedupChangedCalledOnNewDelivery(t *testing.T) { // acknowledged no-ops — each used to deploy the watched branch's current // state with an unrelated changed-file list. func TestWebhookHandler_OnlyWatchedBranchPushes(t *testing.T) { - secret := "s3cret" cases := []struct { name string body string @@ -237,20 +349,18 @@ func TestWebhookHandler_OnlyWatchedBranchPushes(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - triggered := false - handler := newWebhookHandler(webhookHandlerConfig{ - secret: secret, - branch: "main", - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { triggered = true }, - }) - body := []byte(tc.body) - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.body)) - req.Header.Set("X-Hub-Signature-256", githubSign(secret, body)) - rec := httptest.NewRecorder() - handler(rec, req) - if triggered != tc.want { - t.Errorf("triggered = %v, want %v (status %d)", triggered, tc.want, rec.Code) + run := newCountingRun() + handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) + postSigned(t, handler, "s3cret", "", tc.body) + if tc.want { + run.waitCall(t) + } + run.waitIdle(t, queue) + if got := run.count() > 0; got != tc.want { + t.Errorf("deploy ran = %v, want %v", got, tc.want) + } + if got := len(ledger.byKind(autodeploy.AdmissionKindAdmitted)) > 0; got != tc.want { + t.Errorf("admitted = %v, want %v", got, tc.want) } }) } @@ -260,40 +370,25 @@ func TestWebhookHandler_OnlyWatchedBranchPushes(t *testing.T) { // captured signed body under a FRESH delivery ID used to bypass the dedup. // Dedup must be keyed on the authenticated content digest. func TestWebhookHandler_ContentReplayRejected(t *testing.T) { - secret := "s3cret" - body := []byte(`{"ref":"refs/heads/main"}`) - triggerCount := 0 - handler := newWebhookHandler(webhookHandlerConfig{ - secret: secret, - branch: "main", - dedup: autodeploy.NewDeliveryDedup(), - trigger: func(_ []string, _ bool) { - triggerCount++ - }, - }) + run := newCountingRun() + handler, _, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) + body := `{"ref":"refs/heads/main"}` - send := func(delivery string) *httptest.ResponseRecorder { - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(body))) - req.Header.Set("X-Hub-Signature-256", githubSign(secret, body)) - if delivery != "" { - req.Header.Set("X-GitHub-Delivery", delivery) - } - rec := httptest.NewRecorder() - handler(rec, req) - return rec + if rec := postSigned(t, handler, "s3cret", "delivery-1", body); rec.Code != http.StatusOK { + t.Fatalf("first delivery status = %d, want 200", rec.Code) } - - send("delivery-1") + run.waitCall(t) // Same signed body, different delivery ID → replay, no second deploy. - if rec := send("delivery-2"); rec.Code != http.StatusOK { + if rec := postSigned(t, handler, "s3cret", "delivery-2", body); rec.Code != http.StatusOK { t.Errorf("content replay should be a 200 no-op, got %d", rec.Code) } // Same signed body, NO delivery header at all → still a replay. - if rec := send(""); rec.Code != http.StatusOK { + if rec := postSigned(t, handler, "s3cret", "", body); rec.Code != http.StatusOK { t.Errorf("headerless content replay should be a 200 no-op, got %d", rec.Code) } - if triggerCount != 1 { - t.Errorf("trigger called %d times for one unique signed body, want 1", triggerCount) + run.waitIdle(t, queue) + if run.count() != 1 { + t.Errorf("deploy ran %d times for one unique signed body, want 1", run.count()) } } @@ -303,33 +398,22 @@ func TestWebhookHandler_ContentReplayRejected(t *testing.T) { // a different signed body is a new event, and only content dedup decides // replays. func TestWebhookHandler_ReusedDeliveryIDDifferentContentNotSuppressed(t *testing.T) { - secret := "s3cret" - body1 := []byte(`{"ref":"refs/heads/main","after":"aaaa"}`) - body2 := []byte(`{"ref":"refs/heads/main","after":"bbbb"}`) - triggerCount := 0 - handler := newWebhookHandler(webhookHandlerConfig{ - secret: secret, - dedup: autodeploy.NewDeliveryDedup(), - logf: func(string, ...any) {}, - trigger: func(_ []string, _ bool) { - triggerCount++ - }, - }) - post := func(body []byte) { - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(body))) - req.Header.Set("X-Hub-Signature-256", githubSign(secret, body)) - req.Header.Set("X-GitHub-Delivery", "same-delivery-id") - handler(httptest.NewRecorder(), req) - } - post(body1) - post(body2) - if triggerCount != 2 { - t.Errorf("distinct authenticated content under a reused delivery ID must both deploy, got %d", triggerCount) + run := newCountingRun() + handler, _, queue := newAdmissionStack("s3cret", "", "myapp", run.run) + + postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"aaaa"}`) + run.waitCall(t) + postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"bbbb"}`) + run.waitCall(t) + run.waitIdle(t, queue) + if run.count() != 2 { + t.Errorf("distinct authenticated content under a reused delivery ID must both deploy, got %d", run.count()) } // The SAME content replays to a no-op regardless of the header. - post(body1) - if triggerCount != 2 { - t.Errorf("replayed content must be a no-op, got %d", triggerCount) + postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"aaaa"}`) + run.waitIdle(t, queue) + if run.count() != 2 { + t.Errorf("replayed content must be a no-op, got %d", run.count()) } } From b7e030d75e6f5b4c72b6b20e5f4a79621183752c Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:20:27 -0700 Subject: [PATCH 10/12] docs: changelog v0.1.36 (preview identity, webhook admission durability, engine-triggered schedules) --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8050fbc..8fab40c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,45 @@ All notable changes to teploy are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [0.1.36] - 2026-09-22 + +### Fixed + +- **Preview environments no longer collide across branch names.** + Branches that sanitize to the same slug (`feature/login` and + `feature-login`) used to share one preview record — the second + deploy destroyed the first's state, container and route. Previews + now carry canonical IDs (`-p-`, derived from the app and + the FULL branch ref) through state files, container names, network + aliases, Caddy routes and domains; the slug is only a display + prefix. Existing slug-keyed records are adopted when their stored + full branch matches; a genuine collision surfaces an explicit error + naming both branches instead of guessing or deleting. +- **Webhook deliveries during a running deploy were acknowledged, then + silently dropped.** The deploy lock returns immediately when already + held — it does not queue — so every push arriving mid-deploy got a + 200 and then nothing. Admission is now durable before the + acknowledgment (an fsync'd append-only ledger), with one bounded + newest-wins pending slot per app (older pending deliveries are + marked superseded, never piled up in goroutines), and a listener + restart resumes admitted-but-unprocessed work exactly once. + Persistence failure refuses with 503 + Retry-After instead of + acknowledging something that is not durable. + +### Changed + +- **Scheduled redeploys run through the deploy engine.** The cron + script no longer reconstructs the container from `docker inspect` + with its own stop/rm/run (no lock, no health gate, no release + record, no rollback, a stop-to-start downtime window). It keeps the + cheap digest pre-check and, when the digest moved, invokes the new + server-side `teploy autodeploy redeploy` — the same fenced, + health-gated deploy path as the webhook listener. `schedule` gains + `--branch`, installs the server binary, and verifies it supports the + command. **Re-run `teploy autodeploy schedule` on existing apps to + upgrade an installed script**; until then the old script keeps its + previous behavior. + ## [0.1.35] - 2026-09-22 ### Fixed From 793f0182bd8edd23c9000f4007ca4c57a19be7df Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:48:47 -0700 Subject: [PATCH 11/12] Audit: known_hosts algorithm-coverage gap reads as 'key mismatch' (from ship's delivery provisioning) --- AUDIT_OPEN.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 66922f6..a52c900 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1172,3 +1172,21 @@ at base — left alone). non-goal here (the register's stranding posture) and stays open with the graceful-shutdown/bounded-admission remainder of A37/T33 (listener scope, signal-time drain of the worker). + +## Finding from live use (2026-09-22) — SSH host-key algorithm coverage reads as "key mismatch" + +Reported from teploy-ship's S14 trusted-copy provisioning (third wave, +85f44bf receipt). A known_hosts carrying only the host's ed25519 line makes +every connection fail with `ssh: handshake failed: knownhosts: key mismatch` +when the negotiated connection presents a different algorithm (this host +also has rsa + ecdsa host keys). The operator-facing failure names neither +the algorithm presented nor the algorithms on file, so it reads as a MITM +alarm rather than the coverage gap it is — the natural first reaction +(re-scan with `-t ed25519`, per most docs) is exactly what produces the +state. + +Status: OPEN (ergonomics/diagnostics, not correctness — failing closed is +right). Candidate fix: include the presented key type and the on-file types +in the error, or document "scan without -t" in the error string. Found +while provisioning the ship delivery worker; worked around by scanning all +algorithms. From c0efd2669703ebe6e1a825bcb67eec80426f3786 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:59:49 -0700 Subject: [PATCH 12/12] fix(preview,autodeploy,cli): commit-pinned webhook builds + blue/green previews + standalone prune (C02/C06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Webhook deliveries now deploy the AUTHENTICATED commit (payload after/ checkout_sha): fetch-verify-reset with a loud error naming both commits when the sha is gone — never a silent tip fallback; the pin rides the admission ledger through supersede and crash-resume. Previews update blue/green: version-suffixed candidate + its own network alias, readiness gate, route switch, then predecessor retirement (strictly ordered, mutation-verified); a failed candidate leaves the predecessor serving. teploy preview prune enumerates both eras across all apps (cron-able, idempotent, TTL 72h default at create/update). Probe stays 5/5. --- AUDIT_OPEN.md | 186 +++++++++++-- internal/autodeploy/ledger.go | 18 +- internal/autodeploy/paths.go | 54 ++++ internal/autodeploy/paths_test.go | 39 +++ internal/cli/autodeploy.go | 2 +- internal/cli/autodeploy_admission_test.go | 40 +++ internal/cli/autodeploy_serve.go | 86 +++++- internal/cli/autodeploy_serve_test.go | 72 ++++- internal/cli/fetch_checkout_test.go | 156 +++++++++++ internal/cli/preview.go | 43 +-- internal/preview/lifecycle_test.go | 307 ++++++++++++++++++++++ internal/preview/preview.go | 252 +++++++++++++++--- internal/preview/preview_test.go | 11 +- 13 files changed, 1168 insertions(+), 98 deletions(-) create mode 100644 internal/cli/fetch_checkout_test.go create mode 100644 internal/preview/lifecycle_test.go diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index a52c900..689e985 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -160,8 +160,11 @@ defect could corrupt data today. over). `accessory verify-backup` already proves archives in scratch; the orchestrated boundary is new lifecycle surface. - F39 — Crontab edit under a host-side flock. -- F40 — Pinning webhook builds to the payload's exact commit (fetch + - ancestry policy). +- F40 — Pinning webhook builds to the payload's exact commit: LANDED + 2026-09-22 (see the C02 commit-pinned builds slice at the bottom — + fetch + verify + reset to payload.after/checkout_sha, loud refusal when + the commit is unfetchable). The standing F42/A37 remainder (queue + durability beyond the ledger, listener scope) is unchanged. - F42 — Durable webhook queue with per-app workers (in-proc model is bounded by the per-app lock + content dedup). - F43 — Listener scoped to a private address/Unix socket reachable only @@ -374,7 +377,8 @@ into each rather than duplicated as new work items. schema + private workspaces + protected credentials). Architectural. - TCL-47 — engine auth adapters + S3 session tokens. Medium; with F37. - TCL-48 — F42 (durable webhook queue). -- TCL-49 — F40 (pin webhook builds to the event's commit). +- TCL-49 — F40 (pin webhook builds to the event commit): LANDED + 2026-09-22 (C02 commit-pinned builds slice, bottom). - TCL-50 — F60 (complete-plan fingerprint vs display digest). - TCL-51 — RESOLVED 2026-09-18 with F57's opt-in (family section at the bottom). @@ -615,7 +619,8 @@ all packages ok. No push performed. credential-file plumbing shared with the engine images. - A34 — F42 durable webhook queue (ack-before-durable-job remains; A36 closed the dedup-race half). -- A35 — F40 webhook build pinning to the event commit. +- A35 — F40 webhook build pinning to the event commit: LANDED 2026-09-22 + (C02 commit-pinned builds slice, bottom). - A37 — F43 listener scope + operational bounds (graceful shutdown, bounded admission) — the durable queue (A34) is the prerequisite for honest shutdown semantics. @@ -721,8 +726,9 @@ on success. readiness (generation-scoped aliases need F04's handoff). - T24 — A34: durable webhook job queue (ack-before-durable-job remains; A36's content dedup + T26's persisted routing cover the routing halves). -- T25 — A35: webhook builds fetch the watched branch HEAD, not the - authenticated payload commit (fetch + worktree pinning design). +- T25 — A35: webhook builds fetched the watched branch HEAD, not the + authenticated payload commit. LANDED 2026-09-22 (C02 commit-pinned + builds slice, bottom). - T28 — NEW deferral: the scheduled-redeploy cron script is a separate forked deployment engine (no lock, health gate, route/state/metadata commit). Unifying it behind the real deploy engine is the fix; whether @@ -1046,12 +1052,14 @@ Branch-match → all three ambiguity tests fail (adopted instead of refusing). Both reverted; gates after revert: `go vet ./...` clean, `go test ./... -race` all packages ok, gofmt clean on touched files. -**Remaining C06 scope (explicitly NOT in this slice)** — preview -lifecycle behavior (old preview serving until the new one is ready — -destroy-before-recreate stays as-is; expiry timer/automation beyond the -existing deploy-piggyback prune; config propagation through a preview -profile; network/secret isolation between previews) and any Dash-side -changes. +**Remaining C06 scope after this and the 2026-09-22 lifecycle slice (see +the bottom section)** — preview-profile config propagation through a +preview (image build args, env surface per preview), network/secret +isolation between previews, and the enforcement TIMER (pruning now CAN be +cron'd via `teploy preview prune`; nothing schedules it server-side) and +any Dash-side changes. The destroy-before-recreate lifecycle and the +deploy-piggyback-only TTL enforcement recorded above were closed by the +2026-09-22 lifecycle slice at the bottom. ## Product programme slice (2026-09-22) — C02: scheduled redeploys run through the engine @@ -1155,15 +1163,13 @@ torn-tail/corruption parsing, fold/newest/seed units. Gates: `go vet touched files (deploy.go/secret_audit.go/update_test.go were unformatted at base — left alone). -**Remaining C02 scope (explicitly NOT in this slice):** +**Remaining C02 scope (explicitly NOT in that slice):** -- **Commit-pinned builds — still open, verified**: `triggerAutoDeploy` - fetches and resets to `origin/` TIP (autodeploy_serve.go - `git fetch origin && git reset --hard origin/`), not the - authenticated payload's `after` commit — the tip IS pinned to the - remote ref at deploy time, but a push landing between event and fetch - deploys the newer commit under the older event's admission. Full - pinning = F40/A35/T25 (fetch + worktree checkout of the event commit). +- **Commit-pinned builds — LANDED 2026-09-22** as its own slice (see the + C02 commit-pinned builds section at the bottom): the fetch now resets to + the authenticated `payload.after`/`checkout_sha` commit, and an + unfetchable commit fails loudly naming both commits. Closes F40/A35/T25 + and the T25 half of that section's recon. - **Dash/CI trigger convergence** — teploy-dash and CI-triggered deploys do not go through this admission path; converging them onto the ledger + queue (or the engine trigger generally) is cross-repo work. @@ -1190,3 +1196,143 @@ right). Candidate fix: include the presented key type and the on-file types in the error, or document "scan without -t" in the error string. Found while provisioning the ship delivery worker; worked around by scanning all algorithms. + +## Programme slice (2026-09-22, later still) — C02: commit-pinned builds + +Closes the F40/TCL-49/A35/T25 family and the remaining-scope bullet of the +admission-durability slice above. Base revision `b7e030d`; changes left +uncommitted for review. + +**Design:** + +- `autodeploy.PushCommit(body)` (internal/autodeploy/paths.go) extracts the + commit the authenticated push names as the branch's new head — GitLab's + `checkout_sha` preferred, else `after` (GitHub/Gitea/Forgejo). Returns "" + for non-push shapes, tag refs (whose "after" is the tag object), explicit + deletions, the all-zero deletion marker, and malformed hashes (40/64 + lowercase hex required): "" means "deploy the tip and say so", never + "pin to garbage". +- `fetchCheckout` (internal/cli/autodeploy_serve.go, extracted from + triggerAutoDeploy) — tip mode renders the historical fetch+reset unchanged + and states "Deploying tip of "; commit mode fetches the branch, + best-effort fetches the SHA itself (pulls force-pushed-away commits on + servers that allow SHA fetches — GitHub/GitLab do; failure fine), + VERIFIES presence with `git cat-file -e '^{commit}'`, then + `git reset --hard ''` — exact args asserted in tests. An unfetchable + commit FAILS LOUDLY naming the authenticated commit, the branch, and the + branch's current tip (`git rev-parse origin/`), and never resets + the worktree — never a silent tip fallback. Output states + "Deploying from delivery (branch )". +- Threading: the handler parses the commit once per delivery and binds it + to the durable admission (new `AdmissionRecord.Commit`, carried through + superseded/processed transitions), the bounded queue's single worker + passes it to the trigger, and restart resume re-triggers PINNED to the + ledger-recorded commit. `autodeploy redeploy` (the scheduled path) passes + "" — tip mode with the explicit tip output; `triggerAutoDeploy` grew the + commit parameter (empty = tip). + +**Evidence** — TDD: behavioral red first after a pure-behavior-preserving +extraction of fetchCheckout (commit-pinned test failed "not implemented +yet"; unfetchable test failed with no error; handler-threading test failed +with record/trigger commit ""). Ping/tag no-op coverage +(TestWebhookHandler_OnlyWatchedBranchPushes) verified unchanged. New +coverage: PushCommit provider shapes (GitHub/GitLab/deletion/zeros/ +malformed/64-hex), exact fetch/verify/reset command forms + ordering, tip +mode never running pin commands, unfetchable → error naming both commits + +no reset, handler→ledger→trigger commit threading, no-commit → tip, +resume-carries-commit. Mutation checks (in-place, reverted): +removing the cat-file guard fails the commit-pinned and unfetchable tests +for the intended reason; severing PushCommit's pinning fails the handler +threading test and 3 PushCommit subtests. Gates after revert: +`go vet ./...` clean; `go test ./... -race` all 25 packages ok; gofmt +clean on touched files; contract probes 5/5 PASS. + +**Remaining C02 scope:** Dash/CI trigger convergence (cross-repo); +cancellation propagation (supersede never interrupts a running deploy — +deliberate non-goal with the A37/T33 graceful-shutdown remainder). The +ancestry-policy question (should a tip AHEAD of the authenticated commit +ever deploy it? today: yes, the authenticated commit always wins — that is +what "bound to the authenticated commit" means) is settled by the +programme text, not by config. + +## Programme slice (2026-09-22, later still) — C06: preview lifecycle + +Closes the two recorded C06 lifecycle defects. Base revision `b7e030d`; +changes left uncommitted for review. + +**Blue/green previews (defect: "Preview destroys old preview before +starting new"):** + +- Candidate container name/alias/process carry the version + (`-preview-p--`; RunConfig.Name set explicitly), so + each generation has its OWN docker network alias — the stable route can + point at exactly one generation (a shared alias round-robins between + predecessor and candidate the moment both run; SetRoute's doc contract + asks for a specific container name as upstream, which preview now + honors). +- Update order: allocate port → (same-version: `docker rename` the + running predecessor aside `-replaced`, the engine's pattern) → + start candidate → inspect internal port → READINESS GATE → SetRoute + under the STABLE key `-preview-p-` with the CANDIDATE as + upstream → rewrite the record → only then stop+remove the predecessor + (+ remove a legacy-era route key when the adopted record had one; the + canonical key is repointed, not removed). Canonical ID, state path, + route key, and domain are unchanged across updates. +- Readiness gate mirrors internal/deploy/health.go's probe shape scoped to + preview's own executor (no engine import): curl against the candidate's + localhost-published port — 200 ready, 404/3xx → TCP-connect fallback, + bounded by 30s/1s defaults (test-shortened knobs). Main deploys gate the + same way; previews failing a dead image loudly is the consistent + posture. +- On candidate failure (start, port inspect, health, route): stop+remove + the CANDIDATE, rename a renamed-aside predecessor back under its + recorded name, leave the predecessor running/routed/recorded, and the + error names the failed candidate. Same-version updates are the engine's + compromise: the alias is shared for the brief window between candidate + start and predecessor retirement (recorded below as residual). + +**TTL enforcement (defect: "Preview expiration cleanup occurs when another +preview deploy is invoked"):** + +- `Manager.Prune` is the one shared prune core (per-preview outcome + lines; failures warn and continue); `Manager.PruneAll` enumerates + `/deployments/*/previews` across ALL apps and runs that core per app — + canonical and legacy eras alike (enumeration is over files; Destroy + adopts legacy records by full-Branch match as before). Idempotent by + construction; touches nothing outside the previews directories and the + artifacts records name. +- `teploy preview prune` now runs PruneAll (help text says so; connects + via the cwd teploy.yml's server — the file identifies the target, the + prune is not app-scoped). The deploy piggyback calls the SAME + `Manager.Prune` core for its own app. The TTL field is the record's + absolute `ExpiresAt`, default 72h documented on DeployConfig.TTL and + State.ExpiresAt, applied on create AND refreshed on update. + +**Evidence** — TDD: behavioral red recorded before implementation +(ordering test showed stop=5 < switch=12 < run=17 — the defect live in +the call log; failed-candidate deployed "successfully" with no gate; +same-version had no rename-aside; PruneAll stub pruned 0). New coverage: +blue/green ordering via mock call-log indexes (run < reload < stop, rm +after stop, record names the candidate under the stable route key), +failed candidate (predecessor container/route/record byte-identical, +candidate cleaned up, error names candidate + health), same-version +rename-aside + retirement after the healthy candidate holds the name +(no reload expected — a byte-identical Caddyfile block is a deliberate +no-op skip in caddy.mutate), prune exactness across two apps and both +eras + non-preview state.json untouched + idempotent second run (zero +docker commands), TTL default = CreatedAt+72h. All pre-existing preview +tests (coexistence, adoption, ambiguity, prune fixtures) pass unchanged. +Mutation checks (in-place, reverted): moving retirement ahead of the +route switch fails the ordering test with stop-before-switch; making +Prune skip ID-less records fails the era test (pruned 1, want 2) AND the +pre-existing legacy-fixture prune test. Gates: `go vet ./...` clean; +`go test ./... -race` all 25 packages ok; gofmt clean on touched files; +contract probes 5/5 PASS. + +**Remaining C06 scope (residual):** preview-profile config propagation +(per-preview env/build-arg surface), network/secret isolation between +previews (candidates share the `teploy` network today, like all app +containers), the enforcement TIMER (nothing server-side schedules +pruning — `preview prune` is cron-able but teploy ships no daemon, by +design), the same-version shared-alias window above, and Dash-side +changes. diff --git a/internal/autodeploy/ledger.go b/internal/autodeploy/ledger.go index 3733cb3..073e8c5 100644 --- a/internal/autodeploy/ledger.go +++ b/internal/autodeploy/ledger.go @@ -43,12 +43,18 @@ const ( // (ID/Digest/App/Branch/Received) are carried on every record so a folded // line is self-describing; transition kinds add their own fields. type AdmissionRecord struct { - Kind string `json:"kind"` - ID string `json:"id"` - Delivery string `json:"delivery,omitempty"` // provider delivery header, metadata only (A36) - Digest string `json:"digest"` // hex sha256 of the AUTHENTICATED body - App string `json:"app"` - Branch string `json:"branch,omitempty"` + Kind string `json:"kind"` + ID string `json:"id"` + Delivery string `json:"delivery,omitempty"` // provider delivery header, metadata only (A36) + Digest string `json:"digest"` // hex sha256 of the AUTHENTICATED body + App string `json:"app"` + Branch string `json:"branch,omitempty"` + // Commit is the commit the authenticated push named as the branch's new + // head (payload after/checkout_sha; see PushCommit). Carried through + // superseded/processed transitions so restart resume re-triggers the + // delivery PINNED to its commit, not the moving tip. Empty when the + // payload carried no usable commit (tip deploy). + Commit string `json:"commit,omitempty"` Received time.Time `json:"received"` SupersededBy string `json:"superseded_by,omitempty"` At time.Time `json:"at,omitempty"` // transition time (superseded/processed) diff --git a/internal/autodeploy/paths.go b/internal/autodeploy/paths.go index be64554..261adb8 100644 --- a/internal/autodeploy/paths.go +++ b/internal/autodeploy/paths.go @@ -27,6 +27,7 @@ type pushPayload struct { Ref string `json:"ref"` Deleted *bool `json:"deleted"` After string `json:"after"` + CheckoutSHA string `json:"checkout_sha"` } // PushEvent classifies an authenticated webhook body against the branch this @@ -56,6 +57,59 @@ func PushEvent(body []byte, branch string) (ok bool) { return p.Ref == "refs/heads/"+branch || strings.TrimPrefix(p.Ref, "refs/heads/") == branch } +// PushCommit returns the commit the authenticated push event names as the +// new head of the pushed branch (C02 commit pinning): GitLab's checkout_sha +// when present, else after (GitHub, Gitea/Forgejo). The value is what the +// deploy must be pinned to — NOT the branch tip at fetch time. +// +// Empty when the payload carries no usable commit: a non-push shape (the +// caller has already filtered with PushEvent), a branch deletion, the +// all-zero deletion marker, or a malformed hash. An empty result means +// "deploy the tip and say so", never "pin to garbage". +func PushCommit(body []byte) string { + var p pushPayload + if err := json.Unmarshal(body, &p); err != nil { + return "" + } + if p.Deleted != nil && *p.Deleted { + return "" + } + // No ref = ping/non-push; a tag ref's "after" is the tag object, not a + // branch head — neither pins a branch deploy. + if p.Ref == "" || strings.HasPrefix(p.Ref, "refs/tags/") { + return "" + } + for _, c := range []string{p.CheckoutSHA, p.After} { + if isCommitHash(c) { + return c + } + } + return "" +} + +// isCommitHash reports whether s is a well-formed git object id as providers +// send them in push payloads: 40 lowercase hex (SHA-1 repos) or 64 lowercase +// hex (SHA-256 repos), and not the all-zero deletion marker. +func isCommitHash(s string) bool { + if len(s) != 40 && len(s) != 64 { + return false + } + allZero := true + for _, r := range s { + switch { + case r >= '0' && r <= '9': + if r != '0' { + allZero = false + } + case r >= 'a' && r <= 'f': + allZero = false + default: + return false + } + } + return !allZero +} + // githubCommitCap is the number of commits GitHub includes in a push event // payload; more than this and the commits array is truncated, so the file // list is incomplete and we must fail open. diff --git a/internal/autodeploy/paths_test.go b/internal/autodeploy/paths_test.go index c664e26..74bccee 100644 --- a/internal/autodeploy/paths_test.go +++ b/internal/autodeploy/paths_test.go @@ -105,3 +105,42 @@ func TestChangedFilesEmptyOrJunk(t *testing.T) { t.Fatal("unparseable → known=false") } } + +// TestPushCommit (C02): the commit a push payload authenticates as the +// branch's new head — GitLab's checkout_sha preferred, else after +// (GitHub/Gitea/Forgejo). Non-push shapes, deletions, all-zero deletion +// markers, and malformed hashes return "" so the caller deploys the tip and +// says so, never pins to garbage. +func TestPushCommit(t *testing.T) { + const sha = "0123456789abcdef0123456789abcdef01234567" + const sha2 = "fedcba9876543210fedcba9876543210fedcba98" + const zeros = "0000000000000000000000000000000000000000" + cases := []struct { + name string + body string + want string + }{ + {"github after", `{"ref":"refs/heads/main","after":"` + sha + `"}`, sha}, + {"gitlab checkout_sha preferred", `{"ref":"refs/heads/main","checkout_sha":"` + sha + `","after":"` + sha2 + `"}`, sha}, + {"gitlab empty checkout_sha falls to after", `{"ref":"refs/heads/main","checkout_sha":"","after":"` + sha2 + `"}`, sha2}, + {"ping", `{}`, ""}, + {"tag push", `{"ref":"refs/tags/v1.0.0","after":"` + sha + `"}`, ""}, + {"branch deletion marker", `{"ref":"refs/heads/main","deleted":true,"after":"` + sha + `"}`, ""}, + {"all-zero after (deletion)", `{"ref":"refs/heads/main","after":"` + zeros + `"}`, ""}, + {"malformed hash ignored", `{"ref":"refs/heads/main","after":"deadbeef"}`, ""}, + {"non-hex ignored", `{"ref":"refs/heads/main","after":"zzz456789abcdef0123456789abcdef01234567"}`, ""}, + {"unparseable body", `not json`, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := PushCommit([]byte(tc.body)); got != tc.want { + t.Errorf("PushCommit(%s) = %q, want %q", tc.name, got, tc.want) + } + }) + } + // sha256-object-id repos send 64-hex hashes — those must pin too. + const sha256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + if got := PushCommit([]byte(`{"ref":"refs/heads/main","after":"` + sha256 + `"}`)); got != sha256 { + t.Errorf("PushCommit(64-hex) = %q, want %q", got, sha256) + } +} diff --git a/internal/cli/autodeploy.go b/internal/cli/autodeploy.go index cdf275e..b052808 100644 --- a/internal/cli/autodeploy.go +++ b/internal/cli/autodeploy.go @@ -374,7 +374,7 @@ func runAutoDeployRedeploy(app, branch string, strictEnv bool) error { executor := ssh.NewLocalExecutor() defer executor.Close() - return triggerAutoDeploy(ctx, executor, app, branch, autodeploy.BuildDir(app), os.Stdout, nil, false, strictEnv) + return triggerAutoDeploy(ctx, executor, app, branch, autodeploy.BuildDir(app), "", os.Stdout, nil, false, strictEnv) } func newAutoDeployUnscheduleCmd(flags *Flags) *cobra.Command { diff --git a/internal/cli/autodeploy_admission_test.go b/internal/cli/autodeploy_admission_test.go index e3fca16..5af719f 100644 --- a/internal/cli/autodeploy_admission_test.go +++ b/internal/cli/autodeploy_admission_test.go @@ -549,3 +549,43 @@ func TestAdmission_LedgerFileDurability(t *testing.T) { t.Errorf("ledger mode = %v, want 0600", info.Mode().Perm()) } } + +// TestAdmission_ResumeCarriesCommit (C02): the authenticated commit is +// durable in the ledger, so a crash-resumed admission re-triggers PINNED to +// its delivery's commit rather than the moving branch tip. +func TestAdmission_ResumeCarriesCommit(t *testing.T) { + dir := t.TempDir() + ledgerPath := filepath.Join(dir, ".autodeploy-ledger.jsonl") + fileLedger, err := autodeploy.OpenLedger(ledgerPath) + if err != nil { + t.Fatal(err) + } + const sha = "0123456789abcdef0123456789abcdef01234567" + if err := fileLedger.Append(autodeploy.AdmissionRecord{ + Kind: autodeploy.AdmissionKindAdmitted, ID: "id-pin", Delivery: "del-1", Digest: "dddd", + App: "myapp", Branch: "main", Commit: sha, Received: time.Now().UTC().Add(-time.Minute), + }); err != nil { + t.Fatal(err) + } + fileLedger.Close() + + var gotCommit string + var gotMu sync.Mutex + run := func(_ []string, _ bool, commit string) { + gotMu.Lock() + gotCommit = commit + gotMu.Unlock() + } + resumeLedger := &memLedger{} + queue := newAdmissionQueue(resumeLedger, run, func(string, ...any) {}) + if err := resumeAdmissions(ledgerPath, "myapp", resumeLedger, queue, autodeploy.NewDeliveryDedup(), func(string, ...any) {}); err != nil { + t.Fatal(err) + } + waitQueueIdle(t, queue) + + gotMu.Lock() + defer gotMu.Unlock() + if gotCommit != sha { + t.Errorf("resumed deploy pinned to commit %q, want %q (the ledger-recorded commit, not the tip)", gotCommit, sha) + } +} diff --git a/internal/cli/autodeploy_serve.go b/internal/cli/autodeploy_serve.go index 6560aee..3bd9ba5 100644 --- a/internal/cli/autodeploy_serve.go +++ b/internal/cli/autodeploy_serve.go @@ -126,10 +126,10 @@ func runAutoDeployServe(app, branch string, port int, strictEnv bool) error { // Bounded queueing: one worker, one newest-wins pending slot. The // deploy itself runs in the worker — never a goroutine per delivery. - queue := newAdmissionQueue(ledger, func(changedFiles []string, filesKnown bool) { + queue := newAdmissionQueue(ledger, func(changedFiles []string, filesKnown bool, commit string) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - if err := triggerAutoDeploy(ctx, executor, app, branch, buildDir, out, changedFiles, filesKnown, strictEnv); err != nil { + if err := triggerAutoDeploy(ctx, executor, app, branch, buildDir, commit, out, changedFiles, filesKnown, strictEnv); err != nil { logf("deploy failed: %v", err) } else { logf("deploy complete") @@ -324,6 +324,11 @@ func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { // tag/ping payload) means the deploy step must not skip. changedFiles, filesKnown := autodeploy.ChangedFiles(body) + // Bind the deploy to the AUTHENTICATED COMMIT (C02): the payload's + // after/checkout_sha names the exact commit this event built; the + // fetch pins the checkout to it instead of the moving tip. + commit := autodeploy.PushCommit(body) + // DURABLE ADMISSION (C02): the record is fsynced BEFORE the 200. // A crash immediately after the response still leaves the admitted // delivery discoverable by restart resume. @@ -334,6 +339,7 @@ func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { Digest: digest, App: cfg.app, Branch: cfg.branch, + Commit: commit, Received: time.Now().UTC(), } if err := cfg.ledger.Append(rec); err != nil { @@ -386,7 +392,9 @@ const ( ) // queuedAdmission is PENDING WORK AS A RECORD, never a blocked goroutine: -// the single worker picks it up when the running deploy finishes. +// the single worker picks it up when the running deploy finishes. The +// authenticated commit rides on the record (rec.Commit), so a resumed +// admission stays pinned to its delivery's commit across a restart. type queuedAdmission struct { rec autodeploy.AdmissionRecord changedFiles []string @@ -404,11 +412,11 @@ type admissionQueue struct { pending *queuedAdmission ledger autodeploy.LedgerAppender - run func(changedFiles []string, filesKnown bool) + run func(changedFiles []string, filesKnown bool, commit string) logf func(format string, args ...any) } -func newAdmissionQueue(ledger autodeploy.LedgerAppender, run func(changedFiles []string, filesKnown bool), logf func(format string, args ...any)) *admissionQueue { +func newAdmissionQueue(ledger autodeploy.LedgerAppender, run func(changedFiles []string, filesKnown bool, commit string), logf func(format string, args ...any)) *admissionQueue { return &admissionQueue{ledger: ledger, run: run, logf: logf} } @@ -452,7 +460,7 @@ func (q *admissionQueue) worker() { q.mu.Unlock() if q.run != nil { - q.run(item.changedFiles, item.filesKnown) + q.run(item.changedFiles, item.filesKnown, item.rec.Commit) } q.markProcessed(item.rec) } @@ -469,6 +477,7 @@ func (q *admissionQueue) markSupersededLocked(old autodeploy.AdmissionRecord, by Digest: old.Digest, App: old.App, Branch: old.Branch, + Commit: old.Commit, Received: old.Received, SupersededBy: byID, At: time.Now().UTC(), @@ -490,6 +499,7 @@ func (q *admissionQueue) markProcessed(rec autodeploy.AdmissionRecord) { Digest: rec.Digest, App: rec.App, Branch: rec.Branch, + Commit: rec.Commit, Received: rec.Received, At: time.Now().UTC(), }) @@ -541,6 +551,7 @@ func resumeAdmissions(ledgerPath, app string, ledger autodeploy.LedgerAppender, Digest: p.Digest, App: p.App, Branch: p.Branch, + Commit: p.Commit, Received: p.Received, SupersededBy: newest.ID, At: time.Now().UTC(), @@ -568,7 +579,11 @@ func resumeAdmissions(ledgerPath, app string, ledger autodeploy.LedgerAppender, // needs credentials already configured for the server's user, or it's // skipped with a warning), so this can still fail on a server that was // never successfully cloned. -func triggerAutoDeploy(ctx context.Context, executor ssh.Executor, app, branch, buildDir string, out io.Writer, changedFiles []string, filesKnown, strictEnv bool) error { +// +// commit pins the build to the commit the webhook delivery authenticated +// (payload after/checkout_sha — C02); empty deploys the branch tip (the +// scheduled-redeploy path, which has no event to pin to). +func triggerAutoDeploy(ctx context.Context, executor ssh.Executor, app, branch, buildDir, commit string, out io.Writer, changedFiles []string, filesKnown, strictEnv bool) error { // The lock's parent must exist before it can be acquired — a server // whose app was never manually deployed has no /deployments/ yet. if err := state.EnsureAppDir(ctx, executor, app); err != nil { @@ -586,11 +601,8 @@ func triggerAutoDeploy(ctx context.Context, executor ssh.Executor, app, branch, if _, err := executor.Run(ctx, "mkdir -p "+ssh.ShellQuote(buildDir)); err != nil { return fmt.Errorf("creating build directory: %w", err) } - fetchCmd := fmt.Sprintf("cd %s && git fetch origin %s && git reset --hard origin/%s", - ssh.ShellQuote(buildDir), ssh.ShellQuote(branch), ssh.ShellQuote(branch)) - if _, err := executor.Run(ctx, fetchCmd); err != nil { - return fmt.Errorf("fetching %s (is %s a valid git checkout with a fetchable 'origin' remote? this must exist before the first webhook-triggered deploy — see `teploy deploy`'s server-build mode, or clone it manually): %w", - branch, buildDir, err) + if err := fetchCheckout(ctx, executor, buildDir, branch, commit, out); err != nil { + return err } appCfg, err := config.LoadApp(buildDir) @@ -697,6 +709,56 @@ func triggerAutoDeploy(ctx context.Context, executor ssh.Executor, app, branch, return deployBuiltImageFenced(ctx, executor, appCfg, image, version, "localhost", false, needsBuild, lk, &att) } +// fetchCheckout advances buildDir's origin and resets the worktree to the +// commit the delivery authenticated, or to the branch tip when no commit is +// known (C02: a delivery for commit A never silently builds whatever the +// moving branch points at by fetch time). The two modes are stated in the +// deploy output so the operator can see which ran. +// +// Pinning strategy: fetch the branch (brings the tip and its reachable +// history), then best-effort fetch the commit SHA itself — on servers that +// allow SHA fetches (GitHub, GitLab) this also pulls commits no longer +// reachable from the tip after a force-push — then VERIFY the commit is +// present as a commit object before resetting to it. When the commit cannot +// be brought in at all (force-pushed away and garbage-collected, or the +// server rejects SHA fetches), the deploy fails loudly naming both the +// authenticated commit and where the branch is now; it never silently +// falls back to the tip. +func fetchCheckout(ctx context.Context, executor ssh.Executor, buildDir, branch, commit string, out io.Writer) error { + cd := "cd " + ssh.ShellQuote(buildDir) + " && " + if _, err := executor.Run(ctx, cd+"git fetch origin "+ssh.ShellQuote(branch)); err != nil { + return fmt.Errorf("fetching %s (is %s a valid git checkout with a fetchable 'origin' remote? this must exist before the first webhook-triggered deploy — see `teploy deploy`'s server-build mode, or clone it manually): %w", + branch, buildDir, err) + } + if commit == "" { + fmt.Fprintf(out, "Deploying tip of %s\n", branch) + if _, err := executor.Run(ctx, cd+"git reset --hard "+ssh.ShellQuote("origin/"+branch)); err != nil { + return fmt.Errorf("resetting %s to origin/%s: %w", buildDir, branch, err) + } + return nil + } + + fmt.Fprintf(out, "Deploying %s from delivery (branch %s)\n", commit, branch) + // Best-effort direct fetch of the authenticated commit: failure is + // fine (many servers refuse SHA fetches) — the branch fetch above + // already brought everything reachable from the tip, and existence + // is verified before the reset either way. + _, _ = executor.Run(ctx, cd+"git fetch origin "+ssh.ShellQuote(commit)) + if _, err := executor.Run(ctx, cd+"git cat-file -e "+ssh.ShellQuote(commit+"^{commit}")); err != nil { + tip, tipErr := executor.Run(ctx, cd+"git rev-parse "+ssh.ShellQuote("origin/"+branch)) + tip = strings.TrimSpace(tip) + if tipErr != nil || tip == "" { + tip = "" + } + return fmt.Errorf("the delivery's authenticated commit %s cannot be fetched from origin (branch %s is now at %s — the commit was force-pushed away or removed); refusing to deploy the moved tip instead. Re-push the commit or trigger a fresh deploy: %w", + commit, branch, tip, err) + } + if _, err := executor.Run(ctx, cd+"git reset --hard "+ssh.ShellQuote(commit)); err != nil { + return fmt.Errorf("resetting %s to authenticated commit %s: %w", buildDir, commit, err) + } + return nil +} + // resolveTLSFromRoot returns a COPY of tls with relative cert/key paths // resolved against root (audit T31) — the resident autodeploy process runs // under systemd with no WorkingDirectory, so relative paths must never be diff --git a/internal/cli/autodeploy_serve_test.go b/internal/cli/autodeploy_serve_test.go index 1e424e4..6c60c0e 100644 --- a/internal/cli/autodeploy_serve_test.go +++ b/internal/cli/autodeploy_serve_test.go @@ -77,7 +77,7 @@ func (c *countingRun) blocking(block chan struct{}) *countingRun { return c } -func (c *countingRun) run(_ []string, _ bool) { +func (c *countingRun) run(_ []string, _ bool, _ string) { c.mu.Lock() c.calls = append(c.calls, time.Now()) block := c.block @@ -119,7 +119,7 @@ func (c *countingRun) waitIdle(t *testing.T, q *admissionQueue) { // newAdmissionStack wires a handler + queue + ledger with an injectable // deploy runner, the shape runAutoDeployServe uses. -func newAdmissionStack(secret, branch, app string, run func([]string, bool)) (http.HandlerFunc, *memLedger, *admissionQueue) { +func newAdmissionStack(secret, branch, app string, run func([]string, bool, string)) (http.HandlerFunc, *memLedger, *admissionQueue) { ledger := &memLedger{} queue := newAdmissionQueue(ledger, run, func(string, ...any) {}) handler := newWebhookHandler(webhookHandlerConfig{ @@ -434,3 +434,71 @@ func TestResolveTLSFromRoot(t *testing.T) { t.Errorf("input TLSConfig mutated: %+v", in) } } + +// TestWebhookHandler_ThreadsCommitToTrigger (C02): the commit the payload +// authenticates must reach BOTH the durable admission record and the deploy +// trigger — the delivery is bound to its commit end to end. +func TestWebhookHandler_ThreadsCommitToTrigger(t *testing.T) { + var gotCommit string + var gotMu sync.Mutex + run := func(_ []string, _ bool, commit string) { + gotMu.Lock() + gotCommit = commit + gotMu.Unlock() + } + handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run) + + const sha = "0123456789abcdef0123456789abcdef01234567" + postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"`+sha+`"}`) + waitQueueIdle(t, queue) + + admitted := ledger.byKind(autodeploy.AdmissionKindAdmitted) + if len(admitted) != 1 { + t.Fatalf("admitted records = %d, want 1", len(admitted)) + } + if admitted[0].Commit != sha { + t.Errorf("admission record commit = %q, want %q", admitted[0].Commit, sha) + } + gotMu.Lock() + defer gotMu.Unlock() + if gotCommit != sha { + t.Errorf("deploy trigger received commit %q, want %q", gotCommit, sha) + } +} + +// A delivery with NO usable commit still deploys — pinned to nothing (tip), +// stated as such to the trigger. +func TestWebhookHandler_NoCommitMeansTip(t *testing.T) { + var gotCommit string + var gotMu sync.Mutex + run := func(_ []string, _ bool, commit string) { + gotMu.Lock() + gotCommit = commit + gotMu.Unlock() + } + handler, _, queue := newAdmissionStack("s3cret", "main", "myapp", run) + + postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) + waitQueueIdle(t, queue) + + gotMu.Lock() + defer gotMu.Unlock() + if gotCommit != "" { + t.Errorf("payload without a commit pinned trigger to %q, want empty (tip)", gotCommit) + } +} + +// waitQueueIdle blocks until the admission queue has no worker and no +// pending slot (for tests whose run func is not a countingRun). +func waitQueueIdle(t *testing.T, q *admissionQueue) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + workerLive, pending := q.snapshot() + if !workerLive && pending == nil { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatal("queue did not drain to idle within timeout") +} diff --git a/internal/cli/fetch_checkout_test.go b/internal/cli/fetch_checkout_test.go new file mode 100644 index 0000000..94e4f9a --- /dev/null +++ b/internal/cli/fetch_checkout_test.go @@ -0,0 +1,156 @@ +package cli + +// C02 commit-pinned builds: the webhook's fetch must check out the commit +// the delivery AUTHENTICATED (payload after/checkout_sha), not the branch +// tip at fetch time. If the branch moved between push and fetch, the +// delivery's commit still builds; if the commit is gone (force-pushed away, +// deleted), the deploy fails loudly naming both commits — never a silent +// fallback to the tip. + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/ssh" +) + +var errUnfetchable = errors.New("exit status 128") + +const ( + testBuildDir = "/deployments/myapp/build" + testCommit = "0123456789abcdef0123456789abcdef01234567" + testMovedTip = "fedcba9876543210fedcba9876543210fedcba98" + gitQuotedDir = "'/deployments/myapp/build'" + gitQuotedMain = "'main'" +) + +// fetchMocks answers every git command fetchCheckout issues; individual +// tests override cat-file/rev-parse with failures where needed. Matches are +// command prefixes, so they carry the "cd && " lead-in. +func fetchMocks(extra ...ssh.MockCommand) []ssh.MockCommand { + const lead = "cd " + gitQuotedDir + " && " + mocks := []ssh.MockCommand{ + {Match: lead + "git fetch origin", Output: ""}, + {Match: lead + "git cat-file -e", Output: ""}, + {Match: lead + "git rev-parse", Output: testMovedTip}, + {Match: lead + "git reset --hard", Output: ""}, + } + return append(mocks, extra...) +} + +func callSequence(mock *ssh.MockExecutor, needles ...string) []int { + indexes := make([]int, len(needles)) + for i, needle := range needles { + indexes[i] = -1 + for j, call := range mock.Calls { + if indexes[i] == -1 && strings.Contains(call, needle) { + indexes[i] = j + } + } + } + return indexes +} + +// An authenticated commit pins the checkout: the exact commit is fetched, +// verified present as a commit object, and the worktree resets to IT — +// never to origin/. +func TestFetchCheckout_CommitPinned(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", fetchMocks()...) + var buf bytes.Buffer + if err := fetchCheckout(context.Background(), mock, testBuildDir, "main", testCommit, &buf); err != nil { + t.Fatalf("fetchCheckout(commit): %v", err) + } + + // Exact command forms (quoting included). + for _, want := range []string{ + "cd " + gitQuotedDir + " && git fetch origin " + gitQuotedMain, + "cd " + gitQuotedDir + " && git fetch origin '" + testCommit + "'", + "cd " + gitQuotedDir + " && git cat-file -e '" + testCommit + "^{commit}'", + "cd " + gitQuotedDir + " && git reset --hard '" + testCommit + "'", + } { + found := false + for _, call := range mock.Calls { + if call == want { + found = true + break + } + } + if !found { + t.Errorf("missing exact command %q, calls: %v", want, mock.Calls) + } + } + // The reset must target the COMMIT, never the moving tip. + for _, call := range mock.Calls { + if strings.Contains(call, "git reset --hard") && strings.Contains(call, "origin/main") { + t.Errorf("commit-pinned checkout reset to the branch tip: %q", call) + } + } + // Ordering: fetch → (sha fetch) → verify → reset. + seq := callSequence(mock, "git fetch origin "+gitQuotedMain, "git fetch origin '"+testCommit+"'", + "git cat-file -e", "git reset --hard") + for i := 1; i < len(seq); i++ { + if seq[i] <= seq[i-1] { + t.Errorf("command ordering wrong (indexes %v, calls %v)", seq, mock.Calls) + } + } + if out := buf.String(); !strings.Contains(out, "Deploying "+testCommit+" from delivery") { + t.Errorf("output must state the pinned deploy, got: %q", out) + } +} + +// Without a commit (scheduled path), the checkout resets to the branch tip +// and says so — and never runs the pinning verification commands. +func TestFetchCheckout_Tip(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", fetchMocks()...) + var buf bytes.Buffer + if err := fetchCheckout(context.Background(), mock, testBuildDir, "main", "", &buf); err != nil { + t.Fatalf("fetchCheckout(tip): %v", err) + } + + for _, call := range mock.Calls { + if strings.Contains(call, "git cat-file") || strings.Contains(call, testCommit) { + t.Errorf("tip deploy must not run commit-pinning commands: %q", call) + } + } + resetFound := false + for _, call := range mock.Calls { + if call == "cd "+gitQuotedDir+" && git reset --hard 'origin/main'" { + resetFound = true + } + } + if !resetFound { + t.Errorf("tip reset command missing, calls: %v", mock.Calls) + } + if out := buf.String(); !strings.Contains(out, "Deploying tip of main") { + t.Errorf("output must state the tip deploy, got: %q", out) + } +} + +// The unfetchable commit (force-pushed away / deleted): FAIL LOUDLY naming +// BOTH commits — the authenticated one and where the branch is now — and +// never reset the worktree. +func TestFetchCheckout_UnfetchableCommitFailsLoudly(t *testing.T) { + const lead = "cd " + gitQuotedDir + " && " + // The cat-file override must come FIRST: the mock matches in + // registration order, and the default success entry would shadow it. + mock := ssh.NewMockExecutor("1.2.3.4", + append([]ssh.MockCommand{{Match: lead + "git cat-file -e", Err: errUnfetchable}}, fetchMocks()...)...) + var buf bytes.Buffer + err := fetchCheckout(context.Background(), mock, testBuildDir, "main", testCommit, &buf) + if err == nil { + t.Fatal("unfetchable commit must fail the deploy, not fall back to the tip") + } + for _, want := range []string{testCommit, testMovedTip, "main"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error must name %q: %v", want, err) + } + } + for _, call := range mock.Calls { + if strings.Contains(call, "git reset --hard") { + t.Errorf("worktree was reset despite the unfetchable commit: %q", call) + } + } +} diff --git a/internal/cli/preview.go b/internal/cli/preview.go index 8a23482..23ff0e4 100644 --- a/internal/cli/preview.go +++ b/internal/cli/preview.go @@ -111,16 +111,17 @@ func runPreviewDeploy(flags *Flags, branch, ttlStr, image string) error { mgr := preview.NewManager(executor, os.Stdout) - // Prune expired previews for this app before deploying a new one. - // Teploy deliberately has no server-side agent/daemon (see CLAUDE.md), - // so nothing else ever enforces preview TTLs — ExpiresAt was being - // written but never checked by anything, letting expired containers - // and Caddy routes leak indefinitely for an app nobody deploys new - // previews for. Piggybacking on the one client-driven action that's - // guaranteed to recur for any team actively using preview environments - // avoids needing a resident process just for this; teams that stop - // using previews stop accumulating them too. Best-effort: a prune - // failure shouldn't block the actual deploy the operator asked for. + // Prune expired previews for this app before deploying a new one + // (the same shared prune core `teploy preview prune` runs across all + // apps — Manager.Prune). Teploy deliberately has no server-side + // agent/daemon (see CLAUDE.md), so nothing else enforces preview TTLs + // on its own — ExpiresAt was being written but never checked by + // anything, letting expired containers and Caddy routes leak + // indefinitely for an app nobody deploys new previews for. The + // standalone `preview prune` (PruneAll) is the cron-able enforcement + // point; this piggyback keeps an actively-used app clean between runs. + // Best-effort: a prune failure shouldn't block the actual deploy the + // operator asked for. if pruned, err := mgr.Prune(ctx, appCfg.App); err != nil { fmt.Printf("Warning: pruning expired previews: %v\n", err) } else if pruned > 0 { @@ -244,12 +245,18 @@ func newPreviewPruneCmd(flags *Flags) *cobra.Command { return &cobra.Command{ Use: "prune", Short: "Remove expired previews", - Long: "Remove expired previews for this app.\n\n" + - "`teploy preview deploy` already runs this automatically before " + - "deploying a new preview, so you don't normally need to run it " + - "by hand — teploy has no server-side agent/daemon, so nothing " + - "else enforces preview TTLs on a schedule. Run this directly if " + - "you want expired previews cleaned up without deploying a new one.", + Long: "Remove expired previews across ALL apps on the target server,\n" + + "enumerating every app's preview records (both the current\n" + + "canonical and the legacy slug-keyed era).\n\n" + + "`teploy preview deploy` prunes that app's expired previews\n" + + "automatically before deploying, but teploy has no server-side\n" + + "agent/daemon — nothing enforces TTLs on a schedule by itself.\n" + + "This command is the standalone enforcement point: run it by hand\n" + + "or from cron to tear down expired previews even for apps nobody\n" + + "is actively deploying. It connects to the server named by the\n" + + "teploy.yml in the current directory and never touches anything\n" + + "outside /deployments//previews and the artifacts those\n" + + "records name.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runPreviewPrune(flags) @@ -273,7 +280,9 @@ func runPreviewPrune(flags *Flags) error { defer executor.Close() mgr := preview.NewManager(executor, os.Stdout) - n, err := mgr.Prune(ctx, appCfg.App) + // PruneAll — the same shared prune core the deploy piggyback uses + // (Manager.Prune), driven across every app on this server. + n, err := mgr.PruneAll(ctx) if err != nil { return err } diff --git a/internal/preview/lifecycle_test.go b/internal/preview/lifecycle_test.go new file mode 100644 index 0000000..1674861 --- /dev/null +++ b/internal/preview/lifecycle_test.go @@ -0,0 +1,307 @@ +package preview + +// C06 preview lifecycle tests: blue/green updates (the predecessor serves +// until the candidate is healthy and routed — never destroyed first) and +// TTL pruning across all apps and both record eras. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// indexOf returns the first call index containing needle, or -1. +func indexOf(calls []string, needle string) int { + for i, c := range calls { + if strings.Contains(c, needle) { + return i + } + } + return -1 +} + +// Blue/green ordering (C06 defect 1): an update must start the candidate, +// health-check it, switch the Caddy route, and only THEN stop+remove the +// predecessor. The predecessor's stop command may never precede the route +// switch — that ordering was the downtime window the main deploy path +// doesn't have. +func TestDeploy_BlueGreenOrdering(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + append(previewDeployMocks(), + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"})...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + n := len(mock.Calls) + mustDeploy(t, mgr, deployCfg(loginBranch, "v2")) + update := mock.Calls[n:] + + predStop := indexOf(update, "docker stop -t 5 'myapp-preview-p-"+loginIDHex+"-v1'") + if predStop == -1 { + t.Fatalf("predecessor v1 was never stopped after the switch — retire step missing, calls: %v", update) + } + switchIdx := indexOf(update, "docker exec caddy caddy reload") + if switchIdx == -1 { + t.Fatalf("no Caddy route switch (reload) during the update, calls: %v", update) + } + candRun := indexOf(update, "docker run") + if candRun == -1 { + t.Fatalf("candidate container never started, calls: %v", update) + } + if !(candRun < switchIdx && switchIdx < predStop) { + t.Errorf("blue/green order violated: candidate run=%d, route switch=%d, predecessor stop=%d (want run < switch < stop)", + candRun, switchIdx, predStop) + } + // The predecessor is removed too, after the switch. + predRm := indexOf(update, "docker rm 'myapp-preview-p-"+loginIDHex+"-v1'") + if predRm == -1 || predRm < predStop { + t.Errorf("predecessor removal must follow its stop (stop=%d rm=%d), calls: %v", predStop, predRm, update) + } + // The record now names the candidate, under the SAME canonical key and + // stable route key. + var s State + if err := json.Unmarshal(mock.Files[previewStatePath("myapp", loginBranch)], &s); err != nil { + t.Fatal(err) + } + if s.Container != "myapp-preview-p-"+loginIDHex+"-v2" { + t.Errorf("record names container %q, want the v2 candidate", s.Container) + } + if s.Route != "myapp-preview-p-"+loginIDHex { + t.Errorf("route key changed across the update: %q", s.Route) + } +} + +// A candidate that fails its health gate leaves the predecessor RUNNING, +// ROUTED, and RECORDED: the only teardown is the failed candidate itself, +// and the error names the candidate that failed. +func TestDeploy_FailedCandidateLeavesPredecessor(t *testing.T) { + // The bundle's default healthy probe must answer "000" (no response) + // for every probe EXCEPT v1's first — a One-shot "200" entry is + // prepended so it wins while it exists; once consumed, the persistent + // "000" takes over and the v2 candidate never becomes ready. + mocks := []ssh.MockCommand{{Match: "curl -s -o /dev/null", Output: "200", Once: true}} + for _, mc := range previewDeployMocks() { + if mc.Match == "curl -s -o /dev/null" { + mc.Output = "000" + } + mocks = append(mocks, mc) + } + mock := ssh.NewMockExecutor("1.2.3.4", mocks...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + mgr.healthTimeout = 150 * time.Millisecond + mgr.healthInterval = 30 * time.Millisecond + + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + recordPath := previewStatePath("myapp", loginBranch) + recordBefore := string(mock.Files[recordPath]) + caddyBefore := string(mock.Files["/deployments/caddy/Caddyfile"]) + n := len(mock.Calls) + + err := mgr.Deploy(context.Background(), deployCfg(loginBranch, "v2")) + if err == nil { + t.Fatal("a candidate that never becomes healthy must fail the deploy") + } + if !strings.Contains(err.Error(), "myapp-preview-p-"+loginIDHex+"-v2") { + t.Errorf("failure must name the candidate container, got: %v", err) + } + if !strings.Contains(strings.ToLower(err.Error()), "health") { + t.Errorf("failure must say the health gate failed, got: %v", err) + } + + update := mock.Calls[n:] + for _, forbidden := range []string{ + "docker stop -t 5 'myapp-preview-p-" + loginIDHex + "-v1'", + "docker rm 'myapp-preview-p-" + loginIDHex + "-v1'", + } { + if indexOf(update, forbidden) != -1 { + t.Errorf("predecessor touched during a failed candidate update: %q ran", forbidden) + } + } + if got := string(mock.Files[recordPath]); got != recordBefore { + t.Errorf("predecessor record was modified:\nbefore: %s\nafter: %s", recordBefore, got) + } + if got := string(mock.Files["/deployments/caddy/Caddyfile"]); got != caddyBefore { + t.Errorf("route was modified during a failed candidate update:\nbefore: %s\nafter: %s", caddyBefore, got) + } + // The failed candidate itself is cleaned up (stopped + removed). + for _, want := range []string{ + "docker stop -t 5 'myapp-preview-p-" + loginIDHex + "-v2'", + "docker rm 'myapp-preview-p-" + loginIDHex + "-v2'", + } { + if indexOf(update, want) == -1 { + t.Errorf("failed candidate must be cleaned up, missing %q in: %v", want, update) + } + } +} + +// Same-version updates keep working: the running predecessor shares the +// candidate's name, so it is renamed aside before the candidate starts and +// retired after the switch (the main engine's same-version pattern). +func TestDeploy_SameVersionUpdate(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + append(previewDeployMocks(), + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, + ssh.MockCommand{Match: "docker rename", Output: ""})...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + n := len(mock.Calls) + mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) + update := mock.Calls[n:] + + rename := indexOf(update, "docker rename 'myapp-preview-p-"+loginIDHex+"-v1' 'myapp-preview-p-"+loginIDHex+"-v1-replaced'") + if rename == -1 { + t.Fatalf("same-version update must rename the predecessor aside first, calls: %v", update) + } + run := indexOf(update, "docker run") + if run < rename { + t.Errorf("candidate started before the rename freed the name (rename=%d run=%d)", rename, run) + } + // A same-version switch needs no Caddyfile edit — the rendered block is + // byte-identical, and mutate() correctly skips no-op writes — so the + // handoff is the NAME: the candidate holds it (healthy, running) before + // the renamed predecessor is retired. + probe := indexOf(update, "curl -s -o /dev/null") + stop := indexOf(update, "docker stop -t 5 'myapp-preview-p-"+loginIDHex+"-v1-replaced'") + if probe == -1 || stop == -1 || stop < probe { + t.Errorf("renamed predecessor must be retired AFTER the healthy candidate holds the name (probe=%d stop=%d)", probe, stop) + } + if stop < run { + t.Errorf("renamed predecessor retired before the candidate started (run=%d stop=%d)", run, stop) + } +} + +// canonicalRecordJSON renders a modern-era record for seeding. +func canonicalRecordJSON(app, idHex, branch, container, domain string, expires time.Time) string { + return fmt.Sprintf(`{"id":%q,"branch":%q,"repo":"github.com/tyler/myapp","route":"%s-preview-p-%s","domain":%q,"port":49200,"container":%q,"image":"myapp:v1","created_at":"2020-01-01T00:00:00Z","expires_at":%q}`, + app+"-p-"+idHex, branch, app, idHex, domain, container, expires.Format(time.RFC3339)) +} + +// Prune removes EXACTLY the expired set across BOTH record eras, and a +// second run is a no-op (idempotent). +func TestPrune_RemovesExactlyExpiredBothErasIdempotent(t *testing.T) { + expiredAt := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + freshAt := time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC) + + // myapp: canonical expired + canonical fresh. + myExpired := previewStatePath("myapp", loginBranch) + myFresh := previewStatePath("myapp", dashBranch) + // otherapp: legacy expired (slug-keyed file) + canonical fresh. + otherExpired := legacyPreviewStatePath("otherapp", "legacy-feature") + otherFresh := previewStatePath("otherapp", "fresh-feature") + + destroyMocks := []ssh.MockCommand{ + // Per-app record listings (PruneAll enumerates apps first). + ssh.MockCommand{Match: "ls -d /deployments/*/previews", Output: "/deployments/myapp/previews\n/deployments/otherapp/previews"}, + ssh.MockCommand{Match: "ls /deployments/myapp/previews/*.json", Output: myExpired + "\n" + myFresh}, + ssh.MockCommand{Match: "ls /deployments/otherapp/previews/*.json", Output: otherExpired + "\n" + otherFresh}, + // Destroy's Caddyfile transaction (see previewDeployMocks). + ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + } + mock := ssh.NewMockExecutor("1.2.3.4", destroyMocks...) + mock.Files[myExpired] = []byte(canonicalRecordJSON("myapp", loginIDHex, loginBranch, + "myapp-preview-p-"+loginIDHex+"-v1", "preview-feature-login-"+loginIDHex+".myapp.com", expiredAt)) + mock.Files[myFresh] = []byte(canonicalRecordJSON("myapp", dashIDHex, dashBranch, + "myapp-preview-p-"+dashIDHex+"-v1", "preview-feature-login-"+dashIDHex+".myapp.com", freshAt)) + mock.Files[otherExpired] = []byte(`{"branch":"legacy-feature","domain":"preview-legacy-feature.otherapp.com","port":49200,"container":"otherapp-preview-legacy-feature-v1","image":"myapp:v1","created_at":"2020-01-01T00:00:00Z","expires_at":"2020-01-02T00:00:00Z"}`) + mock.Files[otherFresh] = []byte(canonicalRecordJSON("otherapp", "abcd1234", "fresh-feature", + "otherapp-preview-p-abcd1234-v1", "preview-fresh-feature-abcd1234.otherapp.com", freshAt)) + // A non-preview deployment resource that must never be touched. + mock.Files["/deployments/myapp/state.json"] = []byte(`{"app":"myapp"}`) + + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + pruned, err := mgr.PruneAll(context.Background()) + if err != nil { + t.Fatalf("PruneAll: %v", err) + } + if pruned != 2 { + t.Fatalf("pruned %d previews, want exactly the 2 expired ones", pruned) + } + for _, path := range []string{myExpired, otherExpired} { + if _, ok := mock.Files[path]; ok { + t.Errorf("expired record still present: %s", path) + } + } + for _, path := range []string{myFresh, otherFresh} { + if _, ok := mock.Files[path]; !ok { + t.Errorf("fresh record removed: %s", path) + } + } + // The expired previews' containers were stopped (canonical + legacy + // both by their stored container names). + for _, container := range []string{ + "docker stop -t 5 'myapp-preview-p-" + loginIDHex + "-v1'", + "docker stop -t 5 'otherapp-preview-legacy-feature-v1'", + } { + if indexOf(mock.Calls, container) == -1 { + t.Errorf("expired preview's container not stopped: %q, calls: %v", container, mock.Calls) + } + } + if indexOf(mock.Calls, "docker stop -t 5 'myapp-preview-p-"+dashIDHex+"-v1'") != -1 { + t.Error("fresh preview's container was stopped") + } + if indexOf(mock.Calls, "/deployments/myapp/state.json") != -1 { + t.Error("a non-preview deployment resource was touched") + } + if _, ok := mock.Files["/deployments/myapp/state.json"]; !ok { + t.Error("non-preview state file removed") + } + + // Idempotent: the second run finds nothing expired and issues no + // docker commands at all. + n := len(mock.Calls) + pruned2, err := mgr.PruneAll(context.Background()) + if err != nil { + t.Fatalf("second PruneAll: %v", err) + } + if pruned2 != 0 { + t.Fatalf("second prune removed %d, want 0 (idempotent)", pruned2) + } + for _, call := range mock.Calls[n:] { + if strings.Contains(call, "docker") { + t.Errorf("idle prune issued docker commands: %q", call) + } + } +} + +// The TTL default is applied at create AND refreshed at update when the +// caller doesn't pass one: ExpiresAt is exactly CreatedAt + 72h. +func TestDeploy_TTLDefaultApplied(t *testing.T) { + mock := ssh.NewMockExecutor("1.2.3.4", + append(previewDeployMocks(), + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"})...) + var buf bytes.Buffer + mgr := NewManager(mock, &buf) + + cfg := deployCfg(loginBranch, "v1") + cfg.TTL = 0 + before := time.Now().UTC() + mustDeploy(t, mgr, cfg) + + var s State + if err := json.Unmarshal(mock.Files[previewStatePath("myapp", loginBranch)], &s); err != nil { + t.Fatal(err) + } + wantTTL := 72 * time.Hour + got := s.ExpiresAt.Sub(s.CreatedAt) + if got != wantTTL { + t.Errorf("record TTL = %v, want the documented default %v", got, wantTTL) + } + if s.CreatedAt.Before(before) { + t.Errorf("CreatedAt %v predates the deploy %v", s.CreatedAt, before) + } +} diff --git a/internal/preview/preview.go b/internal/preview/preview.go index 3efed3f..6e98388 100644 --- a/internal/preview/preview.go +++ b/internal/preview/preview.go @@ -5,9 +5,12 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" + "net" "regexp" + "strconv" "strings" "time" @@ -41,6 +44,11 @@ type State struct { Container string `json:"container"` Image string `json:"image"` CreatedAt time.Time `json:"created_at"` + // ExpiresAt is the record's absolute TTL deadline: written on every + // Deploy as creation/update time + the configured TTL (the documented + // default is 72h — DeployConfig.TTL). Enforcement reads this field; + // `teploy preview prune` (and the deploy piggyback) destroy records + // whose deadline has passed. ExpiresAt time.Time `json:"expires_at"` } @@ -54,7 +62,10 @@ type DeployConfig struct { EnvFile string Env map[string]string Volumes map[string]string - TTL time.Duration // default 72h + // TTL is how long the preview lives; 0 means the documented default of + // 72h. Applied on every Deploy (create AND update — an update refreshes + // the deadline), recorded as the absolute State.ExpiresAt. + TTL time.Duration // Repo is the normalized repo identity recorded in the preview record // (see State.Repo). Empty is allowed: the repo is provenance, not part // of the preview ID. @@ -67,15 +78,22 @@ type Manager struct { docker *docker.Client caddy *caddy.Client out io.Writer + // healthTimeout/healthInterval bound the candidate readiness gate + // (blue/green switch). Defaults set by NewManager; unexported knobs so + // tests can shorten them. + healthTimeout time.Duration + healthInterval time.Duration } // NewManager creates a preview manager. func NewManager(exec ssh.Executor, out io.Writer) *Manager { return &Manager{ - exec: exec, - docker: docker.NewClient(exec), - caddy: caddy.NewClient(exec), - out: out, + exec: exec, + docker: docker.NewClient(exec), + caddy: caddy.NewClient(exec), + out: out, + healthTimeout: 30 * time.Second, + healthInterval: time.Second, } } @@ -282,6 +300,15 @@ func (m *Manager) writeRecord(ctx context.Context, s *State, path string) error } // Deploy creates or updates a preview environment for the given branch. +// +// Updates are blue/green (C06): the candidate starts under a +// version-suffixed container name and network alias, passes a readiness +// gate, and only then takes over the preview's stable Caddy route key — +// the predecessor is stopped and removed AFTER the switch. A candidate +// that fails its gate (or the route switch) leaves the predecessor +// running, routed, and recorded; only the failed candidate is cleaned up. +// The canonical ID, state path, route key, and domain are stable across +// updates — only the upstream container moves. func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { if cfg.TTL == 0 { cfg.TTL = 72 * time.Hour @@ -305,7 +332,7 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { } // The record's live artifacts predate the migration (Route empty, // slug-keyed) — keep them described exactly as they are so the - // destroy below tears down what is actually running. + // retirement below tears down what is actually running. if err := m.writeRecord(ctx, &adopted, previewStatePath(cfg.App, cfg.Branch)); err != nil { return fmt.Errorf("migrating legacy preview record: %w", err) } @@ -314,9 +341,13 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { idHex := previewIDHex(cfg.App, cfg.Branch) domain := previewDomain(cfg.App, cfg.Branch, cfg.Domain) - process := "preview-p-" + idHex - routeApp := cfg.App + "-" + process - containerName := fmt.Sprintf("%s-%s-%s", cfg.App, process, cfg.Version) + // The process (container name component AND network alias) carries the + // version: each candidate gets its own alias, so the stable route can + // point at exactly one generation — a shared alias would round-robin + // between predecessor and candidate the moment both run. + process := "preview-p-" + idHex + "-" + cfg.Version + routeApp := cfg.App + "-preview-p-" + idHex + containerName := cfg.App + "-" + process fmt.Fprintf(m.out, "Deploying preview for branch %q...\n", cfg.Branch) fmt.Fprintf(m.out, " Domain: %s\n", domain) @@ -326,9 +357,6 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { return fmt.Errorf("creating preview directory: %w", err) } - // Destroy existing preview for this branch if it exists. - m.Destroy(ctx, cfg.App, cfg.Branch) - // Allocate port. port, err := m.docker.FindAvailablePort(ctx) if err != nil { @@ -336,7 +364,42 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { } fmt.Fprintf(m.out, " Port: %d\n", port) - // Start container. + // Same-version update: the running predecessor holds the candidate's + // exact name. Rename it aside (it keeps serving under the shared + // alias until the switch — the main engine's _replaced pattern). + predecessorContainer := "" + predecessorRoute := "" + renamedAside := false + if existing != nil { + predecessorContainer = existing.Container + predecessorRoute = previewRouteKey(cfg.App, existing) + if predecessorContainer == containerName { + if _, err := m.exec.Run(ctx, fmt.Sprintf("docker rename %s %s", + ssh.ShellQuote(predecessorContainer), ssh.ShellQuote(predecessorContainer+"-replaced"))); err != nil { + return fmt.Errorf("renaming the running preview %s aside for the same-version update (a leftover -replaced container may need `docker rm` first): %w", predecessorContainer, err) + } + renamedAside = true + predecessorContainer = predecessorContainer + "-replaced" + } + } + + // abortCandidate tears down the failed candidate and puts a renamed + // predecessor back under its recorded name. The predecessor is never + // touched beyond that — it keeps serving. + abortCandidate := func(reason error, format string, args ...any) error { + m.docker.Stop(ctx, containerName, 5) + m.docker.Remove(ctx, containerName) + if renamedAside { + m.exec.Run(ctx, fmt.Sprintf("docker rename %s %s", + ssh.ShellQuote(predecessorContainer), ssh.ShellQuote(containerName))) + } + if reason != nil { + return fmt.Errorf(format+": %w", append(args, reason)...) + } + return fmt.Errorf(format, args...) + } + + // Start the candidate. var envFiles []string if cfg.EnvFile != "" { envFiles = []string{cfg.EnvFile} @@ -345,6 +408,7 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { App: cfg.App, Process: process, Version: cfg.Version, + Name: containerName, Image: cfg.Image, Port: port, EnvFiles: envFiles, @@ -352,28 +416,33 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { Volumes: cfg.Volumes, }) if err != nil { - return fmt.Errorf("starting preview container: %w", err) + return abortCandidate(err, "starting preview container %s", containerName) } - // Set Caddy route for the preview domain. The preview container gets a - // dedicated network alias (cfg.App + "-" + process) via - // docker.RunConfig.Process, which is what we dial here. // Caddy dials the upstream over the docker network, so it needs the - // container's INTERNAL port, not the host-published port (which is what - // `port` is). Passing the host port made Caddy dial a port the container - // isn't listening on inside the network, so every preview route 502'd. + // container's INTERNAL port, not the host-published port (which is + // what `port` is). Passing the host port made Caddy dial a port the + // container isn't listening on inside the network, so every preview + // route 502'd. internalPort, err := m.docker.InternalPort(ctx, containerName) if err != nil { - m.docker.Stop(ctx, containerName, 5) - m.docker.Remove(ctx, containerName) - return fmt.Errorf("inspecting preview container port: %w", err) + return abortCandidate(err, "inspecting preview container %s port", containerName) } - // Preview subdomains use Caddy automatic HTTPS (no custom cert). - if err := m.caddy.SetRoute(ctx, routeApp, domain, routeApp, internalPort, caddy.TLS{}, "", nil, caddy.Firewall{}, caddy.Access{}); err != nil { - // Clean up container on route failure. - m.docker.Stop(ctx, containerName, 5) - m.docker.Remove(ctx, containerName) - return fmt.Errorf("setting preview route: %w", err) + + // Readiness gate: traffic only switches to a candidate that answers. + // Mirrors the deploy engine's probe (HTTP 200 on /health, 404/3xx + // falling back to a TCP check) against the candidate's + // localhost-published port. + if err := m.waitReady(ctx, port); err != nil { + return abortCandidate(err, "preview candidate %s failed its health check — the previous preview is still serving", containerName) + } + + // Switch the preview domain's route to the candidate. The route KEY + // (and with it the canonical identity) is stable; only the upstream + // container moves. Preview subdomains use Caddy automatic HTTPS (no + // custom cert). + if err := m.caddy.SetRoute(ctx, routeApp, domain, containerName, internalPort, caddy.TLS{}, "", nil, caddy.Firewall{}, caddy.Access{}); err != nil { + return abortCandidate(err, "setting preview route — the previous preview is still serving") } // Write state. @@ -394,11 +463,77 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { return fmt.Errorf("writing preview state: %w", err) } + // Retire the predecessor — strictly AFTER the route serves the + // candidate (blue/green: this ordering is the fix; stopping first was + // the downtime window). + if predecessorContainer != "" { + m.docker.Stop(ctx, predecessorContainer, 5) + m.docker.Remove(ctx, predecessorContainer) + } + // A legacy-era predecessor lived under a different route key; that key + // must go. The canonical key was just repointed, so it stays. + if predecessorRoute != "" && predecessorRoute != routeApp { + m.caddy.RemoveRoute(ctx, predecessorRoute) + } + fmt.Fprintf(m.out, " Preview deployed: https://%s\n", domain) fmt.Fprintf(m.out, " Expires: %s\n", state.ExpiresAt.Format(time.RFC3339)) return nil } +// waitReady polls the candidate's localhost-published port until it +// answers, bounded by the manager's health timeout. The probe mirrors the +// deploy engine's readiness check: HTTP 200 on /health is ready; 404 or a +// redirect means the app is listening but has no /health route, and a TCP +// connect counts as ready; anything else retries until the deadline. +func (m *Manager) waitReady(ctx context.Context, port int) error { + deadlineCtx, cancel := context.WithTimeout(ctx, m.healthTimeout) + defer cancel() + for { + if m.probeOnce(deadlineCtx, port) { + return nil + } + select { + case <-deadlineCtx.Done(): + return fmt.Errorf("no response on localhost:%d within %s", port, m.healthTimeout) + case <-time.After(m.healthInterval): + // retry + } + } +} + +// probeOnce performs one health probe attempt against the preview +// candidate (same curl discipline as internal/deploy's checkHealth: one +// quoted --url argument, globoff, no proxy, bounded per-attempt timeouts). +func (m *Manager) probeOnce(ctx context.Context, port int) bool { + if port < 1 || port > 65535 { + return false + } + target := "http://" + net.JoinHostPort("localhost", strconv.Itoa(port)) + "/health" + out, err := m.exec.Run(ctx, fmt.Sprintf( + "curl -s -o /dev/null --noproxy '*' --globoff --connect-timeout 2 --max-time 5 -w '%%{http_code}' --url %s", + ssh.ShellQuote(target))) + if err == nil { + switch code := strings.TrimSpace(out); { + case code == "200": + return true + case code == "404" || strings.HasPrefix(code, "3"): + return m.probeTCP(ctx, port) + } + } + return false +} + +// probeTCP reports whether a TCP connection to localhost:port succeeds — +// the listening-but-no-/health fallback. +func (m *Manager) probeTCP(ctx context.Context, port int) bool { + if port < 1 || port > 65535 { + return false + } + _, err := m.exec.Run(ctx, fmt.Sprintf("bash -c '/dev/null", port)) + return err == nil +} + // List returns all active previews for the app. Records from both the // canonical-ID keys and legacy slug keys are listed; legacy records are // returned unmodified (readers never mutate). @@ -452,7 +587,9 @@ func (m *Manager) Destroy(ctx context.Context, app, branch string) error { return nil } -// Prune removes expired previews. +// Prune removes a single app's expired previews, reporting the outcome of +// each one. This is the shared prune core: the deploy piggyback and the +// standalone all-apps prune (PruneAll) both run exactly this code path. func (m *Manager) Prune(ctx context.Context, app string) (int, error) { previews, err := m.List(ctx, app) if err != nil { @@ -462,13 +599,56 @@ func (m *Manager) Prune(ctx context.Context, app string) (int, error) { now := time.Now().UTC() pruned := 0 for _, p := range previews { - if now.After(p.ExpiresAt) { - if err := m.Destroy(ctx, app, p.Branch); err != nil { - fmt.Fprintf(m.out, "Warning: failed to prune preview %s: %v\n", p.Branch, err) - continue - } - pruned++ + if !now.After(p.ExpiresAt) { + continue } + if err := m.Destroy(ctx, app, p.Branch); err != nil { + fmt.Fprintf(m.out, "Warning: failed to prune preview %s: %v\n", p.Branch, err) + continue + } + fmt.Fprintf(m.out, "Pruned expired preview %q (%s)\n", p.Branch, p.Domain) + pruned++ } return pruned, nil } + +// PruneAll removes expired previews across EVERY app on the target server, +// enumerating each app's preview records (canonical and legacy eras alike +// — List reads whatever files exist under the app's previews directory). +// This is what the standalone `teploy preview prune` runs, so TTL +// enforcement no longer depends on someone deploying a new preview of the +// same app: the command can be cron'd. Idempotent by construction — a +// pruned record is gone, so a second run finds nothing expired. Never +// touches anything outside /deployments//previews and the artifacts +// the records themselves name. +func (m *Manager) PruneAll(ctx context.Context) (int, error) { + out, err := m.exec.Run(ctx, "ls -d /deployments/*/previews 2>/dev/null") + if err != nil && strings.TrimSpace(out) == "" { + // No preview directories at all — nothing to prune. + return 0, nil + } + + var apps []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + rest, ok := strings.CutPrefix(line, deploymentsDir+"/") + if !ok { + continue + } + app, ok := strings.CutSuffix(rest, "/previews") + if ok && app != "" && !strings.Contains(app, "/") { + apps = append(apps, app) + } + } + + total := 0 + var errs []error + for _, app := range apps { + n, err := m.Prune(ctx, app) + if err != nil { + errs = append(errs, fmt.Errorf("app %s: %w", app, err)) + } + total += n + } + return total, errors.Join(errs...) +} diff --git a/internal/preview/preview_test.go b/internal/preview/preview_test.go index ec4251d..9e7ca15 100644 --- a/internal/preview/preview_test.go +++ b/internal/preview/preview_test.go @@ -97,6 +97,7 @@ func TestDeploy(t *testing.T) { ssh.MockCommand{Match: "ss -tln", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, ssh.MockCommand{Match: "docker inspect -f '{{range $p", Output: "80/tcp"}, + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, ssh.MockCommand{Match: "curl -sf http://localhost:2019/config/apps/http/servers/srv0", Output: `{"listen":[":80",":443"]}`}, ssh.MockCommand{Match: "curl -sf -X PATCH", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "curl -sf -X POST http://localhost:2019/config/apps/http/servers/srv0/routes", Output: ""}, @@ -221,16 +222,18 @@ func TestPreviewDomain(t *testing.T) { } // previewDeployMocks is the mock bundle for a full Deploy against a bare -// server: port allocation, container start, and the Caddyfile -// edit/reload/verify transaction (see TestDeploy for the origins of each -// entry). State-file and Caddyfile writes go through the mock's file -// state, so successive deploys observe each other's records and routes. +// server: port allocation, container start, the candidate health probe +// (blue/green readiness gate), and the Caddyfile edit/reload/verify +// transaction (see TestDeploy for the origins of each entry). State-file +// and Caddyfile writes go through the mock's file state, so successive +// deploys observe each other's records and routes. func previewDeployMocks() []ssh.MockCommand { return []ssh.MockCommand{ ssh.MockCommand{Match: "mkdir -p /deployments/myapp/previews", Output: ""}, ssh.MockCommand{Match: "ss -tln", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, ssh.MockCommand{Match: "docker inspect -f '{{range $p", Output: "80/tcp"}, + ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"},