feat(launch): --max-concurrent-auto derives sweep concurrency from real quota

CHANGELOG.md

@@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- **`spawn launch --max-concurrent-auto`** derives the parameter-sweep
+  concurrency ceiling from the account's real AWS quota headroom instead of a
+  user-typed number (#492). The sweep orchestrator's wave mechanism
+  (`pkg/sweep` + `lambda/sweep-orchestrator`) already polls active-instance
+  count and launches `min(available, remaining)` — but until now its ceiling
+  was always a flag the caller had to already know. `--max-concurrent-auto`
+  queries truffle's quota client for headroom (quota minus current usage) per
+  instance family, in the region the sweep is about to launch in, and
+  converts vCPU headroom to an instance count via the real per-type vCPU
+  count (truffle's new `Capabilities.VCPUs`, not a guessed size suffix). A
+  heterogeneous sweep's derived ceiling is the MINIMUM across every distinct
+  (instance type, spot/on-demand) combination present, so a scarce family
+  can't be silently outvoted by a roomier one. Mutually exclusive with
+  `--max-concurrent`.
+
+  Real-world motivation: a 10-shard fleet launch with no concurrency
+  guardrail hit an account's real ceiling (a G/VT Spot quota of 64 vCPUs,
+  already saturated by 8 running `g7e.2xlarge` instances) with zero prior
+  warning — the actual launches then failed with
+  `MaxSpotInstanceCountExceeded`.
+
+  Not yet wired into `spawn resume`'s `--max-concurrent` override, which has
+  the same class of gap; tracked as a follow-up since resuming already has a
+  recorded region to work from and is a smaller retrofit.
+
+  Requires `truffle` v0.49.0 (bumped as part of this change) for
+  `QuotaInfo.SpotUsage` and `Capabilities.VCPUs`.
+
 ## [0.98.0] - 2026-08-07
 
 ### Changed

cmd/launch_flags.go

@@ -104,6 +104,7 @@ var (
 	params                 string
 	cartesian              bool
 	maxConcurrent          int
+	maxConcurrentAuto      bool
 	maxConcurrentPerRegion int
 	launchDelay            string
 	detach                 bool
@@ -292,6 +293,7 @@ func init() {
 	launchCmd.Flags().StringVar(&params, "params", "", "Inline JSON parameters for sweep")
 	launchCmd.Flags().BoolVar(&cartesian, "cartesian", false, "Generate cartesian product of parameter lists")
 	launchCmd.Flags().IntVar(&maxConcurrent, "max-concurrent", 0, "Max instances running simultaneously (0 = unlimited)")
+	launchCmd.Flags().BoolVar(&maxConcurrentAuto, "max-concurrent-auto", false, "Derive --max-concurrent from the account's real AWS quota headroom for the sweep's instance type(s)/region, instead of a user-supplied number (spawn#492)")
 	launchCmd.Flags().IntVar(&maxConcurrentPerRegion, "max-concurrent-per-region", 0, "Max instances running simultaneously per region (0 = unlimited)")
 	launchCmd.Flags().StringVar(&launchDelay, "launch-delay", "0s", "Delay between instance launches (e.g., 5s)")
 	launchCmd.Flags().BoolVar(&detach, "detach", false, "Run sweep orchestration in Lambda (auto-enabled for parameter sweeps)")

cmd/launch_sweep.go

@@ -21,6 +21,9 @@
 	if detach && noDetach {
 		return fmt.Errorf("--detach and --no-detach are mutually exclusive")
 	}
+	if maxConcurrentAuto && maxConcurrent > 0 {
+		return fmt.Errorf("--max-concurrent-auto and --max-concurrent are mutually exclusive — pick one")
+	}
 
 	// Validate workflow integration flags
 	if wait && !detach {
@@ -54,8 +57,11 @@
 		fmt.Fprintf(os.Stderr, "   This prevents zombie instances if CLI disconnects.\n")
 		fmt.Fprintf(os.Stderr, "   Resume monitoring with: spawn sweep status <sweep-id>\n")
 
-		// If maxConcurrent is 0 (launch all at once), set a reasonable default
-		if maxConcurrent == 0 {
+		// If maxConcurrent is 0 (launch all at once), set a reasonable default —
+		// unless --max-concurrent-auto will derive a real one below, in which
+		// case this arbitrary fallback would just be overwritten (and its
+		// stderr message would print out of order relative to the derived value).
+		if maxConcurrent == 0 && !maxConcurrentAuto {
 			// Default to number of params or 10, whichever is less
 			defaultConcurrent := len(paramFormat.Params)
 			if defaultConcurrent > 10 {
@@ -89,6 +95,39 @@
 		fmt.Fprintf(os.Stderr, "⚠️  Failed to write sweep ID to file: %v\n", err)
 	}
 
+	// --max-concurrent-auto (#492): resolve region (if not already pinned) and
+	// derive a real quota-backed ceiling BEFORE the detached/non-detached branch
+	// below, since a sweep auto-enables --detach and that branch needs a
+	// concrete maxConcurrent > 0 to take the (cheaper, Lambda-orchestrated) path
+	// rather than falling through to the foreground one.
+	if maxConcurrentAuto {
+		region := baseConfig.Region
+		if region == "" {
+			fmt.Fprintf(os.Stderr, "🌍 No region specified, auto-detecting closest region for --max-concurrent-auto...\n")
+			detectedRegion, derr := detectBestRegion(ctx, baseConfig.InstanceType)
+			if derr != nil {
+				fmt.Fprintf(os.Stderr, "⚠️  Could not auto-detect region: %v\n", derr)
+				region = "us-east-1"
+			} else {
+				region = detectedRegion
+			}
+			// Pin it on baseConfig too, so the rest of this function (and
+			// launchSweepDetached, if we end up there) doesn't re-detect it.
+			baseConfig.Region = region
+			fmt.Fprintf(os.Stderr, "✓ Selected region: %s\n", region)
+		}
+		regionClient, derr := aws.NewClientWithRegion(ctx, region)
+		if derr != nil {
+			return fmt.Errorf("--max-concurrent-auto: create AWS client for %s: %w", region, derr)
+		}
+		derived, derr := resolveAutoMaxConcurrent(ctx, paramFormat, baseConfig, region, regionClient)
+		if derr != nil {
+			return fmt.Errorf("--max-concurrent-auto: %w", derr)
+		}
+		fmt.Fprintf(os.Stderr, "✓ --max-concurrent-auto: derived %d (account quota headroom in %s)\n", derived, region)
+		maxConcurrent = derived
+	}
+
 	fmt.Fprintf(os.Stderr, "\n🧪 Parameter Sweep: %s\n", sweepID)
 	fmt.Fprintf(os.Stderr, "   Parameters: %d\n", len(paramFormat.Params))
 	if maxConcurrent > 0 {

cmd/launch_sweep_quota.go

@@ -0,0 +1,151 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spore-host/spawn/pkg/aws"
+	truffleaws "github.com/spore-host/truffle/pkg/aws"
+	truffleQuotas "github.com/spore-host/truffle/pkg/quotas"
+)
+
+// resolveAutoMaxConcurrent derives a --max-concurrent ceiling from the
+// account's real AWS quota headroom, instead of requiring the caller to
+// already know it (spawn#492). The sweep orchestrator's wave mechanism
+// (pkg/sweep + lambda/sweep-orchestrator) has always existed — it polls
+// active-instance count and launches min(available, remaining) — but its
+// ceiling was purely user-typed, with nothing computing or suggesting a
+// value from the account's actual limits.
+//
+// Real-world motivation: a 10-shard fleet with no concurrency guardrail hit
+// an account's real ceiling (a G/VT Spot quota of 64 vCPUs, saturated by 8
+// already-running g7e.2xlarge instances) with zero warning.
+//
+// region is the sweep's resolved launch region (both the detached and
+// non-detached callers already resolve this before this point — see
+// detectBestRegion). It:
+//
+//  1. Extracts the distinct (instance type, spot) combinations the sweep will
+//     actually launch, from the raw param sets (via buildLaunchConfigFromParams,
+//     the same per-entry-override merge the non-detached path already applies —
+//     see the "instance_type is a per-entry override (#372)" comment there).
+//  2. Queries truffle's quota client once for headroom (quota - current usage)
+//     per family, converting vCPU headroom to an instance count via
+//     Capabilities.VCPUs (truffle#492 — the real EC2 value, not a guessed
+//     size-suffix).
+//  3. Returns the MINIMUM headroom-derived instance count across every
+//     combination present in the sweep — the ceiling must respect the
+//     tightest-fitting quota, not the loosest, or a heterogeneous sweep could
+//     still overrun a scarce family while looking fine against a roomy one.
+//
+// A family this can't get a quota/vCPU answer for (missing credentials,
+// unsupported family, API error) is skipped with a warning rather than
+// aborting the whole sweep — this is a SAFETY DEFAULT, not a hard gate, and a
+// caller that explicitly asked for --max-concurrent=auto should still get a
+// best-effort number rather than an outright failure when only some rows in a
+// mixed sweep are affected.
+//
+// awsClient is an already-constructed client PINNED TO region (the caller has
+// already done this re-pinning for AMI/AZ/identity resolution elsewhere —
+// see the #276 comment on the non-detached path); this function builds
+// truffle clients from its config rather than constructing its own, both to
+// avoid a second client construction and so tests can inject a
+// Substrate-backed client.
+func resolveAutoMaxConcurrent(ctx context.Context, paramFormat *ParamFileFormat, baseConfig *aws.LaunchConfig, region string, awsClient *aws.Client) (int, error) {
+	if region == "" {
+		return 0, fmt.Errorf("auto max-concurrent: no region resolved yet — derive the ceiling AFTER region resolution")
+	}
+
+	combos, err := sweepQuotaCombos(paramFormat, baseConfig)
+	if err != nil {
+		return 0, err
+	}
+
+	quotaClient := truffleQuotas.NewClientFromConfig(awsClient.Config())
+	capsClient := truffleaws.NewClientFromConfig(awsClient.Config())
+
+	info, err := quotaClient.GetQuotas(ctx, region)
+	if err != nil {
+		return 0, fmt.Errorf("auto max-concurrent: quota lookup for %s: %w", region, err)
+	}
+
+	best := -1 // -1 = no combo has produced a usable answer yet
+	var warnings []string
+
+	for _, c := range combos {
+		family := truffleQuotas.GetQuotaFamily(c.instanceType)
+		var quota, usage int32
+		if c.spot {
+			quota, usage = info.Spot[family], info.SpotUsage[family]
+		} else {
+			quota, usage = info.OnDemand[family], info.Usage[family]
+		}
+		headroomVCPUs := quota - usage
+		if headroomVCPUs < 0 {
+			headroomVCPUs = 0
+		}
+
+		caps, err := capsClient.GetCapabilities(ctx, c.instanceType, region)
+		if err != nil || !caps.Found || caps.VCPUs <= 0 {
+			warnings = append(warnings, fmt.Sprintf("%s: vCPU count unavailable, cannot convert quota headroom to an instance count", c.instanceType))
+			continue
+		}
+
+		instances := int(headroomVCPUs / caps.VCPUs)
+		if best == -1 || instances < best {
+			best = instances
+		}
+	}
+
+	for _, w := range warnings {
+		fmt.Fprintf(os.Stderr, "⚠️  --max-concurrent=auto: %s\n", w)
+	}
+
+	if best == -1 {
+		return 0, fmt.Errorf("auto max-concurrent: could not derive a ceiling for any instance type in this sweep — specify --max-concurrent explicitly")
+	}
+	if best == 0 {
+		return 0, fmt.Errorf("auto max-concurrent: account quota headroom is 0 for at least one instance type/family in this sweep — request a quota increase, free up running capacity, or specify --max-concurrent explicitly to override this safety check")
+	}
+	return best, nil
+}
+
+// sweepQuotaCombo is one distinct (instance type, spot-vs-on-demand)
+// combination a sweep will actually launch.
+type sweepQuotaCombo struct {
+	instanceType string
+	spot         bool
+}
+
+// sweepQuotaCombos extracts the distinct (instance type, spot) combinations
+// from a sweep's raw param sets, applying the same defaults-then-per-entry
+// merge (buildLaunchConfigFromParams) and base-config fallback the rest of
+// launchParameterSweep uses — pure, no AWS calls, so it's unit-testable
+// without Substrate.
+func sweepQuotaCombos(paramFormat *ParamFileFormat, baseConfig *aws.LaunchConfig) ([]sweepQuotaCombo, error) {
+	seen := make(map[sweepQuotaCombo]bool)
+	var combos []sweepQuotaCombo
+	for i, paramSet := range paramFormat.Params {
+		cfg, err := buildLaunchConfigFromParams(paramFormat.Defaults, paramSet, "", "", i, len(paramFormat.Params))
+		if err != nil {
+			return nil, fmt.Errorf("auto max-concurrent: build launch config for parameter set %d: %w", i, err)
+		}
+		instanceType := cfg.InstanceType
+		if instanceType == "" {
+			instanceType = baseConfig.InstanceType
+		}
+		if instanceType == "" {
+			return nil, fmt.Errorf("auto max-concurrent: parameter set %d has no instance_type", i)
+		}
+		c := sweepQuotaCombo{instanceType: instanceType, spot: cfg.Spot || baseConfig.Spot}
+		if !seen[c] {
+			seen[c] = true
+			combos = append(combos, c)
+		}
+	}
+	if len(combos) == 0 {
+		return nil, fmt.Errorf("auto max-concurrent: no parameter sets to derive a ceiling from")
+	}
+	return combos, nil
+}

cmd/launch_sweep_quota_substrate_test.go

@@ -0,0 +1,82 @@
+package cmd
+
+import (
+	"context"
+	"testing"
+
+	"github.com/spore-host/spawn/pkg/aws"
+	"github.com/spore-host/spawn/pkg/testutil"
+)
+
+// TestResolveAutoMaxConcurrent_DerivesFromRealQuota is the spawn#492
+// end-to-end guard: --max-concurrent-auto must derive an instance-count
+// ceiling from a REAL quota answer + a REAL per-instance-type vCPU count, not
+// a guessed one. Substrate seeds the Standard On-Demand quota (L-1216C47A) at
+// 32 vCPUs and models c5.xlarge at 4 vCPUs, so with zero running instances the
+// expected ceiling is exactly 32/4 = 8.
+func TestResolveAutoMaxConcurrent_DerivesFromRealQuota(t *testing.T) {
+	env := testutil.SubstrateServer(t)
+	ctx := context.Background()
+	client := aws.NewClientFromConfig(env.AWSConfig)
+
+	paramFormat := &ParamFileFormat{
+		Params: []map[string]interface{}{
+			{"instance_type": "c5.xlarge"},
+			{"instance_type": "c5.xlarge"},
+		},
+	}
+	baseConfig := &aws.LaunchConfig{}
+
+	got, err := resolveAutoMaxConcurrent(ctx, paramFormat, baseConfig, "us-east-1", client)
+	if err != nil {
+		t.Fatalf("resolveAutoMaxConcurrent: %v", err)
+	}
+	if got != 8 {
+		t.Errorf("derived max-concurrent = %d, want 8 (32 vCPU quota / 4 vCPU per c5.xlarge)", got)
+	}
+}
+
+// TestResolveAutoMaxConcurrent_TightestFamilyWins is the heterogeneous-sweep
+// guard (#492's own ask): a sweep mixing a scarce family with a roomy one must
+// derive its ceiling from the TIGHTEST-fitting family, not the loosest —
+// otherwise a sweep that "looks fine" against one family could still overrun
+// another. p3.2xlarge's family (P) has no seeded Substrate quota, so its
+// headroom is 0 — which correctly makes the WHOLE sweep's derived ceiling 0
+// (and thus an error, safety-first) even though c5.xlarge alone would derive
+// 8. This is deliberate: silently dropping the P-family rows to "derive 8
+// anyway" would let the sweep launch p3.2xlarge instances with no quota
+// headroom verified at all, exactly the blind spot #492 exists to close.
+func TestResolveAutoMaxConcurrent_TightestFamilyWins(t *testing.T) {
+	env := testutil.SubstrateServer(t)
+	ctx := context.Background()
+	client := aws.NewClientFromConfig(env.AWSConfig)
+
+	paramFormat := &ParamFileFormat{
+		Params: []map[string]interface{}{
+			{"instance_type": "c5.xlarge"},  // Standard family: real 32-vCPU quota, headroom for 8
+			{"instance_type": "p3.2xlarge"}, // P family: no seeded quota in Substrate -> headroom 0
+		},
+	}
+	baseConfig := &aws.LaunchConfig{}
+
+	_, err := resolveAutoMaxConcurrent(ctx, paramFormat, baseConfig, "us-east-1", client)
+	if err == nil {
+		t.Fatal("want an error: the P-family row's zero quota headroom must drag the WHOLE sweep's ceiling to 0, not be silently outvoted by c5.xlarge's roomier 8")
+	}
+}
+
+// TestResolveAutoMaxConcurrent_RequiresResolvedRegion guards the ordering
+// invariant this function's doc comment states: it must be called AFTER
+// region resolution, never before.
+func TestResolveAutoMaxConcurrent_RequiresResolvedRegion(t *testing.T) {
+	env := testutil.SubstrateServer(t)
+	ctx := context.Background()
+	client := aws.NewClientFromConfig(env.AWSConfig)
+
+	paramFormat := &ParamFileFormat{
+		Params: []map[string]interface{}{{"instance_type": "c5.xlarge"}},
+	}
+	if _, err := resolveAutoMaxConcurrent(ctx, paramFormat, &aws.LaunchConfig{}, "", client); err == nil {
+		t.Error("want error when region is empty")
+	}
+}