feat(volume): real modal.CloudBucketMount -> S3 mount support (calque#91 Workstream A)

cmd/calque/fleetrun.go

@@ -247,6 +247,18 @@ func fleetRun(o realOpts, shards int) (err error) {
 	// shard shares the SAME resolved mounts, since they all drive the
 	// same picked unit's own body. Computed once, outside the loop.
 	shardVolumeSync, shardVolumeCommit := volumeSpecsForApp(app, o.bucket, rep)
+	// calque#91 Workstream A: the same real modal.CloudBucketMount(...)
+	// wiring realrun.go added — see cloudBucketMountSpecsForApp's doc
+	// comment. Every shard shares the SAME resolved mounts (they all drive
+	// the same picked unit's own body), passed into runShard's own
+	// BootstrapConfig/IAM setup below (D4's dedicated-fallback-instance
+	// path). NOTE: unlike VolumeSync/VolumeCommit, this does NOT currently
+	// reach the D2 shared-worker-pool path (ProvisionFleetWorkers/
+	// buildFleetWorkerBootstrapCommand, internal/pool/fleet_provision.go) —
+	// that path builds its own bootstrap script independent of
+	// calexec.BootstrapConfig and, same as this pre-existing Volume
+	// plumbing, has no CloudBucketMount mounting either; out of scope here.
+	shardCloudBucketMountLines, shardCloudBucketMountBuckets := cloudBucketMountSpecsForApp(app, rep)
 	var wg sync.WaitGroup
 	for i := range shs {
 		if err := calexec.WriteManifestBody(ctx, s3c, calexec.RunLayout{
@@ -380,7 +392,7 @@ func fleetRun(o realOpts, shards int) (err error) {
 				waitForQuotaHeadroom(ctx, cfg, inst, o.region, o.spot, safeRep)
 			}
 			fmt.Fprintf(os.Stderr, "[fleet] shard %d failed (%v); re-driving once on a fresh instance\n", shs[i].ID, shardErrs[i])
-			m, serr := runShard(ctx, s3c, ec2c, spawnClient, o, shs[i], places, pricePerHr, tgt, shardBody, shardHostMode, safeRep, shardVolumeSync, shardVolumeCommit)
+			m, serr := runShard(ctx, s3c, ec2c, spawnClient, o, shs[i], places, pricePerHr, tgt, shardBody, shardHostMode, safeRep, shardVolumeSync, shardVolumeCommit, shardCloudBucketMountLines, shardCloudBucketMountBuckets)
 			measurements[i], shardErrs[i] = m, serr
 			if serr != nil {
 				safeRep.Addf(leak.PrimAcquire, leak.KindSemanticGap, o.model, 0,
@@ -581,7 +593,7 @@ const fleetWorkerIdleTimeout = 1 * time.Minute
 // pointer across shards would race on that mutation).
 func runShard(ctx context.Context, s3c *s3.Client, ec2c *ec2.Client, spawnClient *spawnaws.Client, o realOpts,
 	sh calexec.Shard, places []plan.Placement, pricePerHr float64, baseTgt *target.Target, body calexec.ManifestBody, hostMode bool, rep *syncReport,
-	volumeSync, volumeCommit []calexec.VolumeSyncSpec) (measure.Measurement, error) {
+	volumeSync, volumeCommit []calexec.VolumeSyncSpec, cloudBucketMountLines, cloudBucketMountBuckets []string) (measure.Measurement, error) {
 	shardLayout := calexec.RunLayout{
 		Bucket: o.bucket, ArtifactPfx: "fleet/" + o.runID + "/artifacts",
 		ManifestKey: sh.ManifestKey, ResultPrefix: sh.ResultPrefix, SummaryKey: sh.SummaryKey, LogKey: sh.LogKey,
@@ -593,11 +605,14 @@ func runShard(ctx context.Context, s3c *s3.Client, ec2c *ec2.Client, spawnClient
 		BaseImage: "vllm/vllm-openai:latest", Bucket: o.bucket, ArtifactPrefix: shardLayout.ArtifactPfx,
 		ManifestKey: shardLayout.ManifestKey, WorkerDir: hostWorkerDir, Region: o.region,
 		LogKey: shardLayout.LogKey, HostMode: hostMode, ModelEnv: o.model,
+		CloudBucketMountLines: cloudBucketMountLines,
 	}
 	// calque#148: see realrun.go's identical fix — without this, the
 	// dedicated fallback instance has no credentials for its own
 	// bootstrap's aws s3 cp/sync calls, including its own failure log.
-	iamProfile, err := plan.RealRunInstanceProfile(ctx, spawnClient, o.region, o.bucket)
+	// calque#91 Workstream A: also grants access to the script's OWN
+	// CloudBucketMount bucket(s), if any (cloudBucketMountBuckets).
+	iamProfile, err := plan.RealRunInstanceProfile(ctx, spawnClient, o.region, o.bucket, cloudBucketMountBuckets...)
 	if err != nil {
 		return measure.Measurement{}, fmt.Errorf("shard %d set up IAM instance profile: %w", sh.ID, err)
 	}

cmd/calque/realrun.go

@@ -344,6 +344,14 @@ func realRun(o realOpts) (err error) {
 	// before calque#79) gets an empty slice both ways — byte-for-byte
 	// unchanged behavior.
 	volumeSync, volumeCommit := volumeSpecsForApp(app, o.bucket, rep)
+	// calque#91 Workstream A: a script's REAL modal.CloudBucketMount(...)
+	// mounts (its OWN S3 bucket, mounted live via mountpoint-s3 — NOT
+	// calque's --bucket staging area the way an ordinary Volume is) resolve
+	// into shell lines spliced into the bootstrap script, plus the distinct
+	// bucket names the instance's IAM role needs read/write/list access to.
+	// A script with no CloudBucketMounts (the vast majority) gets an empty
+	// slice both ways — byte-for-byte unchanged behavior.
+	cloudBucketMountLines, cloudBucketMountBuckets := cloudBucketMountSpecsForApp(app, rep)
 	// calque#148, widened: bootstrap.go's host-mode branch ALWAYS
 	// provisions a uv-managed venv now (not just when --pip supplies real
 	// deps), so warmd must ALWAYS invoke that SAME venv's interpreter for
@@ -384,6 +392,7 @@ func realRun(o realOpts) (err error) {
 		PipPackages: o.pipPackages, PythonVersion: o.pythonVersion,
 		StageFiles: o.stageFiles, RegistryRef: registryRef,
 		BuildDockerfile: buildDockerfile, BuildTag: buildTag,
+		CloudBucketMountLines: cloudBucketMountLines,
 	}
 
 	// calque#134/#178: when --script named a real parsed unit, carry its
@@ -423,8 +432,10 @@ func realRun(o realOpts) (err error) {
 	// its own bootstrap script makes — not even for uploading its OWN
 	// bootstrap log on failure, which is why a bootstrap failure on this
 	// path was previously totally silent (no log, no error, just a
-	// timeout at the deadline). Scoped to just this run's own bucket.
-	iamProfile, err := plan.RealRunInstanceProfile(ctx, spawnClient, o.region, o.bucket)
+	// timeout at the deadline). Scoped to just this run's own bucket, plus
+	// (calque#91 Workstream A) any distinct bucket(s) the script's own
+	// resolved CloudBucketMount(s) reference.
+	iamProfile, err := plan.RealRunInstanceProfile(ctx, spawnClient, o.region, o.bucket, cloudBucketMountBuckets...)
 	if err != nil {
 		return fmt.Errorf("set up IAM instance profile: %w", err)
 	}
@@ -560,6 +571,31 @@ func volumeSpecsForApp(app ir.App, bucket string, rep *leak.Report) (sync, commi
 	return sync, commit
 }
 
+// cloudBucketMountSpecsForApp resolves app's REAL modal.CloudBucketMount(...)
+// mounts (calque#91 Workstream A) into the already-rendered shell lines
+// spliced into BootstrapConfig.CloudBucketMountLines, plus the distinct S3
+// bucket names (the SCRIPT'S OWN buckets, not calque's --bucket staging
+// area) the instance's IAM role needs read/write/list access to — mirrors
+// volumeSpecsForApp's factoring (a pure function, no ctx/S3, so it's
+// unit-testable without a real script/S3 client). A script with no
+// CloudBucketMounts (the vast majority) returns (nil, nil) — byte-for-byte
+// the same as the hardcoded nil, nil this replaces.
+func cloudBucketMountSpecsForApp(app ir.App, rep *leak.Report) (lines []string, buckets []string) {
+	mounts := plan.ResolveCloudBucketMounts(app, rep)
+	if len(mounts) == 0 {
+		return nil, nil
+	}
+	lines = plan.MountCommands(mounts)
+	seen := map[string]bool{}
+	for _, m := range mounts {
+		if !seen[m.BucketName] {
+			seen[m.BucketName] = true
+			buckets = append(buckets, m.BucketName)
+		}
+	}
+	return lines, buckets
+}
+
 func emitK(o realOpts, inst string, perItem []float64, enterSec float64, occ calexec.OccupancyRaw, acq plan.Acquired, priceHr float64) error {
 	rates, err := cost.LoadRates(o.ratesFP)
 	if err != nil {

docs/modal-compatibility-matrix.md

@@ -6,6 +6,10 @@ ports to AWS **unchanged**. This document is the single most direct
 answer to "does calque support my script."
 
 **Provenance:**
+- Updated the §E `modal.CloudBucketMount` row (2026-08-14, calque#91
+  Workstream A): moved from ⬜ (not modeled — leak only) to ✅ (a real
+  mountpoint-s3 mount against the script's own S3 bucket). See the row
+  itself for the file-by-file implementation summary.
 - Verified against calque v0.5.0 (2026-08-14) — updated the App-level
   defaults row for calque#174 (image= per-function resolution fix).
   Also fixed the §H `modal.Cron`/`modal.Period` rows (calque#149 doc
@@ -138,7 +142,7 @@ real gap — should not stay this way) · ⬜ not present at all.
 | `modal.Volume.from_name(...)` + `volumes={mount: vol}` | **Not a live shared filesystem** — snapshot-at-container-start, explicit `.commit()`/`.reload()` for cross-container visibility, last-write-wins on concurrent same-file writes (documented, expected data loss). | 🔥 | ✅ maps to a deterministic S3 prefix, real delta-sync before `@enter`, real end-of-run commit write-back. | calque's model (sync-before-run, commit-after-run) matches Modal's snapshot-at-start semantics reasonably well for the common case; **mid-run `.reload()`** (re-sync during execution) is correctly leaked as unreproduced. | — |
 | `.commit()` / `.reload()` call sites | End-of-run persistence / mid-run re-read. | 🔥 (wherever Volumes are used) | ✅ `.commit()` honored as real end-of-run write-back. 🟨 `.reload()` leaked as unreproduced. | — | — |
 | `modal.NetworkFileSystem` (deprecated, being removed) | **Live-shared** filesystem — no commit/reload cycle, closer to EFS/NFS than Volume's snapshot model. | 🧊 (deprecated, Modal steers users to Volume) | ⬜ | If a real script still uses this, calque's Volume→S3-prefix mapping is the WRONG model (S3 has no live-shared-write semantics) — this would need an EFS-shaped mapping instead, not a Volume-shaped one. | [#91](https://github.com/spore-host/calque/issues/91) |
-| `modal.CloudBucketMount` | Direct S3/R2/GCS mount via `mountpoint-s3` — no append writes, no seek+write, must open in truncate mode, no rename. | 🧊 | ⬜ | A script using this directly against real S3 is a DIFFERENT (and more restrictive) primitive than Volume — calque's Volume mapping doesn't cover it. | [#91](https://github.com/spore-host/calque/issues/91) |
+| `modal.CloudBucketMount` | Direct S3/R2/GCS mount via `mountpoint-s3` — no append writes, no seek+write, must open in truncate mode, no rename. | 🧊 | ✅ (calque#91 Workstream A) a real `CloudBucketMount(bucket_name, key_prefix=, read_only=)` used INLINE as a `volumes=` value (the real Modal idiom — constructed directly in the dict, not assigned to a variable first) resolves to a real mountpoint-s3 mount against the SCRIPT'S OWN S3 bucket: `tools/pyast/pyast.py`'s `_cloud_bucket_mount` extracts `bucket_name`/`key_prefix`/`read_only`, `internal/parse/parse.go` decodes them into `ir.Function`/`ir.Class.CloudBucketMounts`, and `internal/plan/cloudbucketmount.go`'s `MountCommands` renders the on-instance `mount-s3` invocation (spliced into the bootstrap script before `@enter` runs, via `internal/exec.BootstrapConfig.CloudBucketMountLines`); `internal/plan.RealRunPolicy`'s `extraBuckets` param grants the instance role read/write/list on that bucket, separate from calque's own `--bucket` staging area. `secret=` is recognized but NOT honored (the instance's own IAM role is used instead) — leaked distinctly. `bucket_endpoint_url=`/`requester_pays=`/`force_path_style=` are NOT supported — leaked distinctly; mounting is against AWS S3 with default settings only. A `bucket_name` that isn't a string literal still falls back to the pre-existing "recognized but not modeled" leak. | R2/GCS-backed CloudBucketMounts (`bucket_endpoint_url=`) are NOT reproduced — AWS S3 only. No live-Modal-managed credential rotation via `secret=`; the instance's own IAM role is the only credential path. | [#91](https://github.com/spore-host/calque/issues/91) (Workstream A closed; NetworkFileSystem is a separate, later workstream) |
 | `modal.Dict` | Distributed KV store, cloudpickle values, 7-day inactivity TTL, capped `.len()` at 100,000. | 🧊 | ⬜ not modeled, but [#151](https://github.com/spore-host/calque/issues/151) closed the failure mode: a bare reference to a module-level `Dict.from_name(...)` constant used to ship verbatim and crash at runtime with a confusing Modal SDK auth error — it's now refused with a clear leak naming the construct instead. | — | [#91](https://github.com/spore-host/calque/issues/91) |
 | `modal.Queue` | FIFO **per-partition only**, 24h partition auto-expiry. | 🧊 | ⬜ not modeled; same [#151](https://github.com/spore-host/calque/issues/151) honest-refusal fix applies to a bare reference to a `Queue.from_name(...)` constant. | — | [#91](https://github.com/spore-host/calque/issues/91) |
 
@@ -312,11 +316,14 @@ a generic "unmodeled arg" message.
     live-verified end-to-end — [#98](https://github.com/spore-host/calque/issues/98)
     (closed).
 12. Lower-priority/rare, still open: `modal.Dict`/`Queue`,
-    `@modal.batched`, `modal.NetworkFileSystem`, `modal.CloudBucketMount`,
+    `@modal.batched`, `modal.NetworkFileSystem`,
     `App.include`/`.deploy`/`.run` lifecycle nuances. (`cloud=` closed
     separately, calque#91's own §C fix; `modal.Cron`/`Period` object-form
     *recognition* also closed under calque#91 — see §H — though actually
-    executing on a schedule remains out of scope.)
+    executing on a schedule remains out of scope. `modal.CloudBucketMount`
+    is now REAL — see §E — closed as calque#91 Workstream A;
+    `modal.NetworkFileSystem` is a separate, larger workstream planned for
+    later, not attempted here.)
     [#91](https://github.com/spore-host/calque/issues/91)
 
 Not individually filed (genuinely low-priority/narrow; revisit if real usage

internal/exec/bootstrap.go

@@ -94,6 +94,16 @@ type BootstrapConfig struct {
 	// content-addressing property internal/image already documents for a
 	// future ECR push path). Ignored when BuildDockerfile is false.
 	BuildTag string
+	// CloudBucketMountLines are already-rendered shell lines (calque#91
+	// Workstream A) that mount every resolved modal.CloudBucketMount(...)
+	// via mountpoint-s3 — the caller (cmd/calque/realrun.go) builds these via
+	// plan.MountCommands(plan.ResolveCloudBucketMounts(app, rep)), so this
+	// package never needs to import internal/plan (no import-cycle risk).
+	// Spliced in right after the artifact sync, before either the HostMode
+	// or docker-mode run invocation — the mount must be live before @enter
+	// runs, in either mode. nil/empty (the default) is a no-op, reproducing
+	// prior behavior byte-for-byte for every script with no CloudBucketMount.
+	CloudBucketMountLines []string
 }
 
 // ecrHostname matches an ECR registry hostname, e.g.
@@ -172,6 +182,14 @@ func (b BootstrapConfig) Command() string {
 		}
 	}
 
+	// calque#91 Workstream A: mount every resolved modal.CloudBucketMount(...)
+	// via mountpoint-s3 BEFORE either the HostMode or docker-mode run
+	// invocation below — the mount must be live before @enter runs, in
+	// either mode. Empty (the default) is a no-op.
+	if len(b.CloudBucketMountLines) > 0 {
+		lines = append(lines, b.CloudBucketMountLines...)
+	}
+
 	if b.HostMode {
 		// Smoke test / real-AWS host-mode: run warmd directly on the host —
 		// no docker, no GPU-container layer. Isolates acquisition +

internal/exec/bootstrap_demo_test.go

@@ -268,3 +268,61 @@ func TestBootstrapCommandDockerModeWithNonECRRegistryRefNoLogin(t *testing.T) {
 		t.Errorf("non-ECR RegistryRef should still be pulled (anonymously); got:\n%s", cmd)
 	}
 }
+
+// TestBootstrapCommandSplicesCloudBucketMountLinesDockerMode (calque#91
+// Workstream A) proves CloudBucketMountLines are spliced into Command()'s
+// output AFTER the artifact sync but BEFORE the docker run invocation —
+// the mount must be live before @enter runs.
+func TestBootstrapCommandSplicesCloudBucketMountLinesDockerMode(t *testing.T) {
+	c := BootstrapConfig{
+		BaseImage: "vllm/vllm-openai:latest", Bucket: "b", ArtifactPrefix: "runs/x/art",
+		ManifestKey: "runs/x/manifest.json", Region: "us-west-2",
+		CloudBucketMountLines: []string{"mkdir -p /data", "mount-s3 my-bucket /data"},
+	}
+	cmd := c.Command()
+	for _, w := range []string{"mkdir -p /data", "mount-s3 my-bucket /data"} {
+		if !strings.Contains(cmd, w) {
+			t.Errorf("missing %q in:\n%s", w, cmd)
+		}
+	}
+	syncIdx := strings.Index(cmd, "aws s3 cp --recursive")
+	mountIdx := strings.Index(cmd, "mount-s3 my-bucket /data")
+	runIdx := strings.Index(cmd, "docker run")
+	if syncIdx == -1 || mountIdx == -1 || runIdx == -1 {
+		t.Fatalf("missing expected markers in:\n%s", cmd)
+	}
+	if syncIdx >= mountIdx || mountIdx >= runIdx {
+		t.Errorf("expected order artifact-sync < cloud-bucket-mount < docker-run; got:\n%s", cmd)
+	}
+}
+
+// TestBootstrapCommandSplicesCloudBucketMountLinesHostMode is the HostMode
+// sibling: the mount must be live before warmd itself runs (host mode has
+// no docker run invocation at all).
+func TestBootstrapCommandSplicesCloudBucketMountLinesHostMode(t *testing.T) {
+	c := BootstrapConfig{
+		Bucket: "b", ArtifactPrefix: "runs/x/art", ManifestKey: "runs/x/manifest.json",
+		Region: "us-west-2", HostMode: true,
+		CloudBucketMountLines: []string{"mkdir -p /data", "mount-s3 my-bucket /data"},
+	}
+	cmd := c.Command()
+	mountIdx := strings.Index(cmd, "mount-s3 my-bucket /data")
+	warmdIdx := strings.Index(cmd, "warmd run --manifest")
+	if mountIdx == -1 || warmdIdx == -1 {
+		t.Fatalf("missing expected markers in:\n%s", cmd)
+	}
+	if mountIdx > warmdIdx {
+		t.Errorf("cloud-bucket-mount lines must run BEFORE warmd; got:\n%s", cmd)
+	}
+}
+
+// TestBootstrapCommandNoCloudBucketMountLinesUnchanged proves the default
+// (empty CloudBucketMountLines) reproduces prior behavior byte-for-byte: no
+// mount-s3 anything appears at all.
+func TestBootstrapCommandNoCloudBucketMountLinesUnchanged(t *testing.T) {
+	c := BootstrapConfig{BaseImage: "vllm/vllm-openai:latest", Bucket: "b", ArtifactPrefix: "runs/x/art", ManifestKey: "runs/x/manifest.json", Region: "us-west-2"}
+	cmd := c.Command()
+	if strings.Contains(cmd, "mount-s3") {
+		t.Errorf("no CloudBucketMountLines set — must not emit any mount-s3 reference; got:\n%s", cmd)
+	}
+}