Support S3 to s3

cmk-data-transfer/.env.example

@@ -1,116 +1,71 @@
-# ============================================================================
-# cmk-data-transfer — configuration
-# Copy to .env and fill in.  .env is git-ignored; never commit real creds.
-#   cp .env.example .env
-# ============================================================================
+# --- Source S3 ---
+SRC_S3_ACCESS_KEY_ID=
+SRC_S3_SECRET_ACCESS_KEY=
+SRC_S3_ENDPOINT=           # e.g. https://s3.us-east-1.amazonaws.com
+SRC_S3_REGION=             # e.g. us-east-1 (optional, depends on provider)
+SRC_S3_BUCKET=
+SRC_S3_PREFIX=             # optional: only copy objects under this prefix
 
-# --- OCI Object Storage credentials (S3-compatible "Customer Secret Key") ---
-# Create under: OCI Console > Profile > Customer Secret Keys.
-# These are the S3-compat access/secret pair, NOT your OCI API signing key.
-OCI_ACCESS_KEY_ID=
-OCI_SECRET_ACCESS_KEY=
+# --- Destination type ---
+# disk : write to a VAST-backed RWX shared disk (default)
+# s3   : write to an S3-compatible destination (e.g. Crusoe S3)
+DEST_TYPE=disk
 
-# --- OCI Object Storage location ---
-# NAMESPACE is the Object Storage namespace (Console > Tenancy details, or
-#   `oci os ns get`). REQUIRED to build the S3-compat endpoint.
-OCI_NAMESPACE=
+# --- Destination S3 (only when DEST_TYPE=s3) ---
+DEST_S3_ACCESS_KEY_ID=
+DEST_S3_SECRET_ACCESS_KEY=
+DEST_S3_ENDPOINT=          # e.g. https://s3.crusoe.ai
+DEST_S3_REGION=
+DEST_S3_BUCKET=
+DEST_S3_PREFIX=
 
-# REGION is the OCI region slug where the BUCKET lives, e.g. us-phoenix-1 /
-# us-ashburn-1. Must be a valid OCI region (not your destination/cloud region);
-# preflight warns if the slug doesn't look like OCI.
-OCI_REGION=us-phoenix-1
-
-# Source bucket and optional prefix (no leading slash on prefix).
-OCI_BUCKET=
-OCI_PREFIX=
-
-# Derived endpoint (leave blank to auto-build from NAMESPACE+REGION):
-#   https://<NAMESPACE>.compat.objectstorage.<REGION>.oraclecloud.com
-OCI_ENDPOINT=
-
-# --- Destination (VAST RWX shared disk) ---
+# --- Destination disk (only when DEST_TYPE=disk) ---
+DEST_PATH=/data/dataset
+DEST_MODE=dynamic           # dynamic | import | nfs
 PVC_NAME=cmk-data-transfer-fs
 PVC_SIZE=1000Ti
 STORAGE_CLASS=crusoe-csi-driver-fs-sc
-# DEST_PATH = WHERE objects are saved on the disk. The disk mounts at /data in
-# every pod, so this is the folder under it (e.g. /data/my-dataset). For an
-# existing disk (DEST_MODE=import/nfs), point it at where on that disk you want
-# the files; rclone copy is idempotent, so an existing partial folder is fine.
-DEST_PATH=/data/dataset
-
-# --- Destination mode: dynamic | import | nfs ---
-#   dynamic : provision a new shared disk via the fs CSI driver (default).
-#   import  : bind an EXISTING disk via a CSI static PV (id+name+serial below).
-#   nfs     : bind an EXISTING disk via an in-tree NFS PV straight to the VAST
-#             DNS endpoint, bypassing the CSI driver. Use this where CSI mounts
-#             time out on an unroutable fallback IP (disk returns no data-path
-#             connectivity fields). Needs only EXISTING_DISK_ID.
-DEST_MODE=dynamic
 
-# Existing-disk identity (find with `crusoe storage disks list -f json`):
-#   import needs all three; nfs needs only EXISTING_DISK_ID.
-# A static PV is created with reclaimPolicy=Retain (your disk is never deleted);
-# PVC_SIZE becomes the PV capacity; STORAGE_CLASS is ignored.
-EXISTING_DISK_ID=                # crusoe disk `id`  (nfs path = /volumes/<id>; CSI volumeHandle)
-EXISTING_DISK_NAME=              # crusoe disk `name`   -> csi.crusoe.ai/disk-name   (import only)
-EXISTING_DISK_SERIAL=            # crusoe disk `serial_number`                       (import only)
+# --- Import existing disk (DEST_MODE=import) ---
+EXISTING_DISK_ID=
+EXISTING_DISK_NAME=
+EXISTING_DISK_SERIAL=
 EXISTING_DISK_FSTYPE=ext4
 
-# NFS-mode endpoint (the VAST data path on CMK):
+# --- NFS mode (DEST_MODE=nfs) ---
 NFS_SERVER=nfs.crusoecloudcompute.com
-NFS_EXPORT_PATH=                 # blank => /volumes/<EXISTING_DISK_ID>
+NFS_EXPORT_PATH=
 NFS_MOUNT_OPTIONS=vers=3,nconnect=16,spread_reads,spread_writes,remoteports=dns
 
-# --- Fleet sizing ---
-# TARGET_GBPS is the single sizing knob — set to your target aggregate GB/s.
-TARGET_GBPS=30
-NUM_NODES=4                       # nodes to spread across
-PODS_PER_NODE=8                   # workers per node (spread evenly). More pods =
-                                  # smaller shards = shorter straggler tail. Raise
-                                  # for shorter tails; if each pod uses many
-                                  # streams, watch node memory.
-NUM_PODS=                         # absolute total override; default = NODES*PER_NODE
+# --- Sizing ---
+TARGET_GBPS=30.0
+NUM_NODES=4
+PODS_PER_NODE=8
+# NUM_PODS=               # absolute override
 INSTANCE_CLASS=s2a
-
-# Node hardware (defaults = s2a.80x). Set for other SKUs/clouds so the sizing
-# model uses the right CPU split, NIC line rate, and BDP.
 NODE_VCPU=80
 NODE_RAM_GIB=676
 NODE_NIC_GBPS=200
-
-# Path RTT (ms) source<->destination region. Drives the BDP / concurrency math.
-RTT_MS=150
-# Conservative per-stream throughput assumption (Mbps) for an untuned
-# intercontinental TCP stream. Lower => more streams provisioned. Tune via sweep.
-PER_STREAM_MBPS=250
-# Safety multiplier on the BDP-derived stream count.
+RTT_MS=150.0
+PER_STREAM_MBPS=250.0
 STREAM_SAFETY=1.5
 
-# --- rclone concurrency (blank => auto-derive per-pod from the BDP sizing) ---
-# Leave TRANSFERS/STREAMS blank to size from TARGET_GBPS + RTT. To pin them, the
-# Crusoe AWS-S3 reference used transfers=40, streams=40, no --checkers (strong
-# throughput + low latency on large files):
-#   RCLONE_TRANSFERS=40 ; RCLONE_MULTI_THREAD_STREAMS=40
-RCLONE_TRANSFERS=                 # parallel files per pod (blank = auto)
-RCLONE_MULTI_THREAD_STREAMS=      # ranged-GET streams per file >cutoff (blank = auto)
+# --- rclone (blank = auto-derive from sizing) ---
+# RCLONE_TRANSFERS=
+# RCLONE_MULTI_THREAD_STREAMS=
 RCLONE_MULTI_THREAD_CUTOFF=256M
-RCLONE_MULTI_THREAD_CHUNK_SIZE=32M  # smaller chunk = shorter per-file straggler tail
-RCLONE_CHECKERS=                  # blank => omit --checkers (rclone default; with
-                                  # --no-traverse there's little to check + it
-                                  # avoids extra HEAD latency)
+RCLONE_MULTI_THREAD_CHUNK_SIZE=32M
+# RCLONE_CHECKERS=
 RCLONE_BUFFER_SIZE=32M
 RCLONE_S3_CHUNK_SIZE=64M
-RCLONE_EXTRA_FLAGS=               # e.g. "--low-level-retries 20 --timeout 120s"
+# RCLONE_EXTRA_FLAGS=
 
-# --- Worker pod resource requests (blank => auto-derived; the auto memory
-# estimate is conservative for high stream counts, which can over-restrict how
-# many pods schedule — set an explicit value when running 40/40). Requests
-# affect scheduling only; with no limits set, actual use can exceed the request.
-WORKER_CPU_REQUEST=
-WORKER_MEM_REQUEST=16Gi
+# --- Worker resources (blank = auto-derive) ---
+# WORKER_CPU_REQUEST=
+# WORKER_MEM_REQUEST=
 
-# --- K8s plumbing ---
+# --- K8s ---
 NAMESPACE=default
-SECRET_NAME=cmk-data-transfer-oci
+SECRET_NAME=cmk-data-transfer-s3
 RCLONE_IMAGE=rclone/rclone:latest
-KUBECONFIG=
+# KUBECONFIG=

cmk-data-transfer/.gitignore

@@ -37,3 +37,4 @@ venv/
 .vscode/
 *.swp
 .claude/
+.superpowers/

cmk-data-transfer/Makefile

@@ -17,7 +17,7 @@ sizing: ## Print the BDP/target sizing plan (no cluster access)
 dry-run: ## Render manifests + run preflight checks, launch nothing
 	$(PY) -m orchestrator.run --dry-run
 
-preflight: ## VAST write-ceiling test (fio, one pod per node) — safe, no egress
+preflight: ## Disk write-ceiling test (fio, one pod per node) — safe, no egress
 	$(PY) preflight/run_fio.py
 
 fio-bench: ## Disk read+write benchmark: fio on every s2a node -> bench/results/
@@ -45,7 +45,7 @@ clean: ## Delete worker + master + fio pods (keep PVC/Secret/data)
 	-kubectl -n $(NAMESPACE) delete pod cmk-data-transfer-master --ignore-not-found
 
 clean-all: clean ## Also delete the Secret (keeps the PVC and downloaded data)
-	-kubectl -n $(NAMESPACE) delete secret cmk-data-transfer-oci --ignore-not-found
+	-kubectl -n $(NAMESPACE) delete secret cmk-data-transfer-s3 --ignore-not-found
 	@echo "PVC and downloaded data left intact. Delete the PVC manually if desired."
 
 # --- defaults for sweep knobs (override on CLI) ---