fix(acme): stop punishing certificate issuance prepared before a DNS cutover

README.md

@@ -783,6 +783,45 @@ root path. Services deployed to other paths on the same host will use the same
 TLS settings as those specified for the root path.
 
 
+### Deploying before a DNS cutover
+
+You can deploy a host before its DNS points at the proxy. Nothing is ordered
+from the certificate authority until the first HTTPS request for that hostname
+arrives, and before the domain resolves here, an HTTP-01 challenge cannot
+succeed — so the proxy checks first, with a cheap request to the domain, that
+it actually routes back here. If it does not, the handshake is refused and no
+order is spent. That matters: Let's Encrypt allows only five failed
+authorizations per hostname per hour, and tripping that limit is what would
+otherwise delay the certificate at the moment of the cutover.
+
+Held domains are re-checked every minute (`--acme-release-probe-interval`, or a
+negative value to switch it off). As soon as the domain starts routing here,
+the hold lifts and the certificate is issued — so the cutover costs about a
+minute, not a backoff step.
+
+To watch it happen, or to see why a certificate has not arrived:
+
+    kamal-proxy domains list
+
+    SERVICE   DOMAIN            CERTIFIED  HOLD                             REMOVAL HELD
+    service1  app.example.com   yes
+    service1  new.example.com   no         2026-08-25 14:32:10 (preflight)
+
+A `preflight` hold means the domain does not point here yet — repoint it and
+the proxy will pick it up on its own. An `acme` hold means the certificate
+authority rejected an order, and lifts the same way. A `rate_limited` hold is
+the authority's own window, and waits it out.
+
+If you know the cause is already fixed and do not want to wait:
+
+    kamal-proxy domains retry new.example.com
+
+Domains covered by a DNS-01 provider skip all of this — see [Wildcard
+Certificates](#wildcard-certificates-dns-01-challenge). DNS-01 validation never
+depends on where a domain points, so those certificates can be issued days
+before a cutover with nothing to check and nothing to wait for.
+
+
 ### On-demand TLS
 
 Instead of specifying a static list of hosts, Kamal Proxy can also obtain TLS
@@ -1041,8 +1080,11 @@ per-domain by default, throttled well under Let's Encrypt's account limits
 (burst of 20 orders, then one per 40s, max 3 in flight). Before a domain's
 first order, the proxy probes `http://<domain>/.kamal-proxy/preflight/<nonce>`
 to verify DNS actually routes here — unreachable domains are quarantined
-(5m, then 15m → 1h → 4h → 24h backoff) without burning an order. Failing
-domains quarantine alone; the rest of a batch is retried once. Renewals reuse
+(5m, then 15m → 1h → 4h → 24h backoff) without burning an order. A held
+domain is re-probed every minute and its hold lifts as soon as it routes
+here, so a backoff step is a ceiling rather than a wait; only a hold the
+certificate authority imposed (rate limiting) runs to its own end time.
+Failing domains quarantine alone; the rest of a batch is retried once. Renewals reuse
 the exact same identifier set (exempt from most rate limits) and pass ARI
 `replaces` where supported. Every renewal re-probes its dynamic members
 first, so a tenant whose DNS moved away after issuance is quarantined out of
@@ -1074,11 +1116,16 @@ the app is down.
 **Inspecting:**
 
 ```bash
-kamal-proxy domains list      # every dynamic domain, cert + quarantine + held status
-kamal-proxy domains stats     # counters: domains, certified, queued, quarantined, held
-kamal-proxy domains refresh   # trigger an immediate re-poll of all sources
+kamal-proxy domains list           # every domain, cert + hold + held-removal status
+kamal-proxy domains stats          # counters: domains, certified, queued, quarantined, held
+kamal-proxy domains refresh        # trigger an immediate re-poll of all sources
+kamal-proxy domains retry <domain> # clear an issuance hold and try again now
 ```
 
+Holds lift on their own once a domain routes back to the proxy — see
+[Deploying before a DNS cutover](#deploying-before-a-dns-cutover). `retry` is
+for when you already know the cause is fixed and would rather not wait.
+
 
 ### Wildcard Certificates (DNS-01 Challenge)
 

internal/cmd/domains.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"maps"
 	"net/rpc"
@@ -20,12 +21,13 @@ func newDomainsCommand() *domainsCommand {
 	domainsCommand := &domainsCommand{}
 	domainsCommand.cmd = &cobra.Command{
 		Use:   "domains",
-		Short: "Inspect and refresh dynamic TLS domains",
+		Short: "Inspect TLS domains and their certificate issuance",
 	}
 
 	domainsCommand.cmd.AddCommand(newDomainsListCommand().cmd)
 	domainsCommand.cmd.AddCommand(newDomainsStatsCommand().cmd)
 	domainsCommand.cmd.AddCommand(newDomainsRefreshCommand().cmd)
+	domainsCommand.cmd.AddCommand(newDomainsRetryCommand().cmd)
 
 	return domainsCommand
 }
@@ -52,7 +54,7 @@ func newDomainsListCommand() *domainsListCommand {
 	domainsListCommand := &domainsListCommand{}
 	domainsListCommand.cmd = &cobra.Command{
 		Use:     "list",
-		Short:   "List dynamic domains by service",
+		Short:   "List dynamic domains and registered hosts by service",
 		RunE:    domainsListCommand.run,
 		Args:    cobra.NoArgs,
 		Aliases: []string{"ls"},
@@ -83,10 +85,7 @@ func (c *domainsListCommand) run(cmd *cobra.Command, args []string) error {
 					certified = "yes"
 				}
 
-				quarantined := ""
-				if entry, ok := response.Quarantine[domain.Domain]; ok {
-					quarantined = entry.Until.Format("2006-01-02 15:04:05")
-				}
+				quarantined := holdDescription(response.Quarantine, domain.Domain)
 
 				held := ""
 				if _, ok := heldRemovals[domain.Domain]; ok {
@@ -97,10 +96,39 @@ func (c *domainsListCommand) run(cmd *cobra.Command, args []string) error {
 			}
 		}
 
+		// Deploy-registered hosts have no domain source to group them under,
+		// but they are the common case — and the one an operator is staring at
+		// during a DNS cutover, wondering why the certificate has not arrived.
+		for _, domain := range slices.Sorted(maps.Keys(response.Registered)) {
+			entry := response.Registered[domain]
+
+			certified := "no"
+			if entry.Certified {
+				certified = "yes"
+			}
+
+			table.AddRow([]string{entry.Service, domain, certified, holdDescription(response.Quarantine, domain), ""})
+		}
+
 		table.Print()
 	})
 }
 
+// holdDescription renders a domain's issuance hold for the listing: when it
+// lifts, and why it is held, since only some kinds lift on their own.
+func holdDescription(quarantine map[string]server.QuarantineStatus, domain string) string {
+	entry, ok := quarantine[domain]
+	if !ok {
+		return ""
+	}
+
+	until := entry.Until.Format("2006-01-02 15:04:05")
+	if entry.Kind == "" {
+		return until
+	}
+	return until + " (" + entry.Kind + ")"
+}
+
 type domainsStatsCommand struct {
 	cmd *cobra.Command
 }
@@ -132,9 +160,18 @@ func (c *domainsStatsCommand) run(cmd *cobra.Command, args []string) error {
 			}
 		}
 
+		registeredCertified := 0
+		for _, entry := range response.Registered {
+			if entry.Certified {
+				registeredCertified++
+			}
+		}
+
 		fmt.Printf("Services with domain sources: %d\n", len(response.Services))
 		fmt.Printf("Dynamic domains:              %d\n", domains)
-		fmt.Printf("Certified:                    %d\n", certified)
+		fmt.Printf("Dynamic certified:            %d\n", certified)
+		fmt.Printf("Registered hosts:             %d\n", len(response.Registered))
+		fmt.Printf("Registered certified:         %d\n", registeredCertified)
 		fmt.Printf("Queued for issuance:          %d\n", response.QueueLength)
 		fmt.Printf("Quarantined:                  %d\n", len(response.Quarantine))
 		fmt.Printf("Held removals:                %d\n", held)
@@ -175,3 +212,61 @@ func (c *domainsRefreshCommand) run(cmd *cobra.Command, args []string) error {
 		return nil
 	})
 }
+
+type domainsRetryCommand struct {
+	cmd *cobra.Command
+	all bool
+}
+
+func newDomainsRetryCommand() *domainsRetryCommand {
+	domainsRetryCommand := &domainsRetryCommand{}
+	domainsRetryCommand.cmd = &cobra.Command{
+		Use:   "retry [domain]",
+		Short: "Clear an issuance hold and try again now",
+		Long: "Clear the issuance hold on a domain and request its certificate again.\n\n" +
+			"Holds normally lift on their own once the domain routes back to this proxy,\n" +
+			"so reach for this when you know the cause is fixed and do not want to wait —\n" +
+			"including for a rate-limit hold, whose window the automatic release respects.",
+		RunE: domainsRetryCommand.run,
+		Args: cobra.MaximumNArgs(1),
+	}
+
+	domainsRetryCommand.cmd.Flags().BoolVar(&domainsRetryCommand.all, "all", false, "Clear every issuance hold")
+
+	return domainsRetryCommand
+}
+
+func (c *domainsRetryCommand) run(cmd *cobra.Command, args []string) error {
+	if len(args) == 0 && !c.all {
+		return errors.New("specify a domain, or --all to clear every hold")
+	}
+	if len(args) > 0 && c.all {
+		return errors.New("specify a domain or --all, not both")
+	}
+
+	domain := ""
+	if len(args) > 0 {
+		domain = args[0]
+	}
+
+	return withRPCClient(globalConfig.SocketPath(), func(client *rpc.Client) error {
+		var cleared int
+
+		err := client.Call("kamal-proxy.DomainsRetry", server.DomainsRetryArgs{Domain: domain}, &cleared)
+		if err != nil {
+			return err
+		}
+
+		switch {
+		case cleared == 0 && domain != "":
+			fmt.Printf("No issuance hold on %s\n", domain)
+		case cleared == 0:
+			fmt.Println("No issuance holds to clear")
+		case cleared == 1:
+			fmt.Println("Cleared 1 issuance hold")
+		default:
+			fmt.Printf("Cleared %d issuance holds\n", cleared)
+		}
+		return nil
+	})
+}

internal/cmd/run.go

@@ -70,6 +70,7 @@ func newRunCommand() *runCommand {
 	runCommand.cmd.Flags().StringSliceVar(&runCommand.acmeDNSProviders, "acme-dns-provider", strings.Split(getEnvString("ACME_DNS_PROVIDER", ""), ","), "DNS provider for DNS-01 challenges (one of: "+providers.ProviderListForHelp()+"). DNS-01 activates only when set: auto detects the provider from environment credentials, none (the default) disables it. Repeatable: zone=provider entries pin a zone to the DNS host that serves it, and one bare entry is the default for unmatched zones")
 	runCommand.cmd.Flags().BoolVar(&globalConfig.ACMEPreferWildcard, "acme-prefer-wildcard", getEnvBool("ACME_PREFER_WILDCARD", true), "Prefer wildcard certificates when DNS provider available")
 	runCommand.cmd.Flags().BoolVar(&globalConfig.ACMEHTTPFallback, "acme-http-fallback", getEnvBool("ACME_HTTP_FALLBACK", true), "Fall back to HTTP-01 challenge if DNS-01 fails")
+	runCommand.cmd.Flags().DurationVar(&globalConfig.ACMEReleaseProbeInterval, "acme-release-probe-interval", 0, "How often to re-probe domains whose issuance is held, so a hold lifts as soon as the domain points here — a DNS cutover then costs one interval instead of a backoff step (default 1m; negative disables)")
 
 	return runCommand
 }
@@ -160,9 +161,10 @@ func (c *runCommand) run(cmd *cobra.Command, args []string) error {
 		router.SetSANCertManager(manager)
 
 		dynamicDomains = server.NewDynamicDomainManager(server.DynamicDomainConfig{
-			StatePath:    globalConfig.DynamicDomainsStatePath(),
-			RefreshToken: os.Getenv("KAMAL_PROXY_REFRESH_TOKEN"),
-			SourceToken:  os.Getenv("KAMAL_PROXY_DOMAINS_TOKEN"),
+			StatePath:            globalConfig.DynamicDomainsStatePath(),
+			RefreshToken:         os.Getenv("KAMAL_PROXY_REFRESH_TOKEN"),
+			SourceToken:          os.Getenv("KAMAL_PROXY_DOMAINS_TOKEN"),
+			ReleaseProbeInterval: globalConfig.ACMEReleaseProbeInterval,
 		}, manager, router)
 
 		router.SetDynamicDomainManager(dynamicDomains)

internal/server/commands.go

@@ -74,6 +74,11 @@ type CertsExportArgs struct {
 	Path string
 }
 
+type DomainsRetryArgs struct {
+	// Domain to clear the hold on. Empty clears every hold.
+	Domain string
+}
+
 type CachePurgeArgs struct {
 	Service    string
 	PathPrefix string
@@ -107,13 +112,29 @@ type DomainsServiceStatus struct {
 type QuarantineStatus struct {
 	Until    time.Time `json:"until"`
 	Failures int       `json:"failures"`
+
+	// Kind is why the domain is held: "preflight" (it did not route back to
+	// this proxy, no order spent), "acme" (the CA rejected an order), or
+	// "rate_limited" (the CA dictated the end time). Only the first two are
+	// lifted early when the domain starts routing here again.
+	Kind string `json:"kind"`
+}
+
+// RegisteredDomainStatus reports the certificate state of a deploy-registered
+// host — one named by `deploy --host`, as opposed to a tenant domain learned
+// from a source.
+type RegisteredDomainStatus struct {
+	Service   string    `json:"service"`
+	Certified bool      `json:"certified"`
+	ExpiresAt time.Time `json:"expires_at,omitempty"`
 }
 
 type DomainsStatusResponse struct {
-	Services     map[string]DomainsServiceStatus `json:"services"`
-	QueueLength  int                             `json:"queue_length"`
-	Quarantine   map[string]QuarantineStatus     `json:"quarantine"`
-	Certificates int                             `json:"certificates"`
+	Services     map[string]DomainsServiceStatus   `json:"services"`
+	QueueLength  int                               `json:"queue_length"`
+	Quarantine   map[string]QuarantineStatus       `json:"quarantine"`
+	Certificates int                               `json:"certificates"`
+	Registered   map[string]RegisteredDomainStatus `json:"registered"`
 }
 
 func NewCommandHandler(router *Router) *CommandHandler {
@@ -270,6 +291,18 @@ func (h *CommandHandler) DomainsStatus(args bool, reply *DomainsStatusResponse)
 	return nil
 }
 
+// DomainsRetry clears issuance holds so a domain can be tried again now. An
+// empty Domain clears every hold.
+func (h *CommandHandler) DomainsRetry(args DomainsRetryArgs, reply *int) error {
+	dynamicDomains := h.router.DynamicDomainManager()
+	if dynamicDomains == nil {
+		return errors.New("dynamic domains are not enabled (start the proxy with --acme-email)")
+	}
+
+	*reply = dynamicDomains.Retry(args.Domain)
+	return nil
+}
+
 func (h *CommandHandler) DomainsRefresh(args bool, reply *int) error {
 	dynamicDomains := h.router.DynamicDomainManager()
 	if dynamicDomains == nil {

internal/server/config.go

@@ -116,6 +116,12 @@ type Config struct {
 	ACMEPreferWildcard bool
 	ACMEHTTPFallback   bool
 
+	// ACMEReleaseProbeInterval is how often a held domain is re-probed so its
+	// hold can be lifted as soon as it routes here again — the difference
+	// between a DNS cutover costing a probe interval and costing a backoff
+	// step. Zero uses the default; negative disables release probing.
+	ACMEReleaseProbeInterval time.Duration
+
 	// ACMEDNSProviderZones maps DNS zones to the provider answering DNS-01
 	// for them, for fleets whose zones live at different DNS hosts.
 	// ACMEDNSProvider stays the default for unmatched zones.

internal/server/domain_failure.go

@@ -24,12 +24,12 @@ const maxConcurrentProbes = 16
 // member otherwise, and the whole set when neither can tell — an
 // unattributable failure holds the entire batch on the quarantine ladder so
 // retries back off instead of looping against ACME rate limits.
-func identifyFailedDomains(err error, domains []string, preflight func(string) error) []string {
+func identifyFailedDomains(err error, domains []string, preflight func(string) error, unprobeable func(string) bool) []string {
 	if failed := failedDomainsFromError(err, domains); len(failed) > 0 {
 		return failed
 	}
 
-	if failed, _ := probeDomains(domains, preflight); len(failed) > 0 {
+	if failed, _ := probeDomains(domains, preflight, unprobeable); len(failed) > 0 {
 		return failed
 	}
 
@@ -38,9 +38,15 @@ func identifyFailedDomains(err error, domains []string, preflight func(string) e
 
 // probeDomains runs the pre-flight probe over a set of domains with bounded
 // concurrency and returns the ones that failed, in input order, with each
-// failure's error. Wildcards are skipped — there is no name to answer on one.
-// A nil probe reports nothing.
-func probeDomains(domains []string, preflight func(string) error) ([]string, map[string]error) {
+// failure's error. A nil probe reports nothing.
+//
+// Domains the probe cannot speak for are skipped rather than failed: a
+// wildcard, which has no name to answer on, and anything unprobeable reports
+// — in practice a zone with a DNS-01 provider, whose order never depends on
+// where the domain points. Probing those and holding them back on failure
+// would punish exactly the case DNS-01 exists to make safe: issuing a
+// certificate before a DNS cutover.
+func probeDomains(domains []string, preflight func(string) error, unprobeable func(string) bool) ([]string, map[string]error) {
 	if preflight == nil {
 		return nil, nil
 	}
@@ -49,7 +55,7 @@ func probeDomains(domains []string, preflight func(string) error) ([]string, map
 	sem := make(chan struct{}, maxConcurrentProbes)
 	var wg sync.WaitGroup
 	for idx, domain := range domains {
-		if strings.HasPrefix(domain, "*.") {
+		if strings.HasPrefix(domain, "*.") || (unprobeable != nil && unprobeable(domain)) {
 			continue
 		}
 		wg.Add(1)

internal/server/domain_failure_test.go

@@ -66,7 +66,7 @@ func TestIdentifyFailedDomains(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.expected, identifyFailedDomains(tt.err, tt.domains, tt.preflight))
+			assert.Equal(t, tt.expected, identifyFailedDomains(tt.err, tt.domains, tt.preflight, nil))
 		})
 	}
 }