diff --git a/cmk-data-transfer/.env.example b/cmk-data-transfer/.env.example index faf3894..3e3ed4b 100644 --- a/cmk-data-transfer/.env.example +++ b/cmk-data-transfer/.env.example @@ -1,116 +1,71 @@ -# ============================================================================ -# cmk-data-transfer — configuration -# Copy to .env and fill in. .env is git-ignored; never commit real creds. -# cp .env.example .env -# ============================================================================ +# --- Source S3 --- +SRC_S3_ACCESS_KEY_ID= +SRC_S3_SECRET_ACCESS_KEY= +SRC_S3_ENDPOINT= # e.g. https://s3.us-east-1.amazonaws.com +SRC_S3_REGION= # e.g. us-east-1 (optional, depends on provider) +SRC_S3_BUCKET= +SRC_S3_PREFIX= # optional: only copy objects under this prefix -# --- OCI Object Storage credentials (S3-compatible "Customer Secret Key") --- -# Create under: OCI Console > Profile > Customer Secret Keys. -# These are the S3-compat access/secret pair, NOT your OCI API signing key. -OCI_ACCESS_KEY_ID= -OCI_SECRET_ACCESS_KEY= +# --- Destination type --- +# disk : write to a VAST-backed RWX shared disk (default) +# s3 : write to an S3-compatible destination (e.g. Crusoe S3) +DEST_TYPE=disk -# --- OCI Object Storage location --- -# NAMESPACE is the Object Storage namespace (Console > Tenancy details, or -# `oci os ns get`). REQUIRED to build the S3-compat endpoint. -OCI_NAMESPACE= +# --- Destination S3 (only when DEST_TYPE=s3) --- +DEST_S3_ACCESS_KEY_ID= +DEST_S3_SECRET_ACCESS_KEY= +DEST_S3_ENDPOINT= # e.g. https://s3.crusoe.ai +DEST_S3_REGION= +DEST_S3_BUCKET= +DEST_S3_PREFIX= -# REGION is the OCI region slug where the BUCKET lives, e.g. us-phoenix-1 / -# us-ashburn-1. Must be a valid OCI region (not your destination/cloud region); -# preflight warns if the slug doesn't look like OCI. -OCI_REGION=us-phoenix-1 - -# Source bucket and optional prefix (no leading slash on prefix). -OCI_BUCKET= -OCI_PREFIX= - -# Derived endpoint (leave blank to auto-build from NAMESPACE+REGION): -# https://.compat.objectstorage..oraclecloud.com -OCI_ENDPOINT= - -# --- Destination (VAST RWX shared disk) --- +# --- Destination disk (only when DEST_TYPE=disk) --- +DEST_PATH=/data/dataset +DEST_MODE=dynamic # dynamic | import | nfs PVC_NAME=cmk-data-transfer-fs PVC_SIZE=1000Ti STORAGE_CLASS=crusoe-csi-driver-fs-sc -# DEST_PATH = WHERE objects are saved on the disk. The disk mounts at /data in -# every pod, so this is the folder under it (e.g. /data/my-dataset). For an -# existing disk (DEST_MODE=import/nfs), point it at where on that disk you want -# the files; rclone copy is idempotent, so an existing partial folder is fine. -DEST_PATH=/data/dataset - -# --- Destination mode: dynamic | import | nfs --- -# dynamic : provision a new shared disk via the fs CSI driver (default). -# import : bind an EXISTING disk via a CSI static PV (id+name+serial below). -# nfs : bind an EXISTING disk via an in-tree NFS PV straight to the VAST -# DNS endpoint, bypassing the CSI driver. Use this where CSI mounts -# time out on an unroutable fallback IP (disk returns no data-path -# connectivity fields). Needs only EXISTING_DISK_ID. -DEST_MODE=dynamic -# Existing-disk identity (find with `crusoe storage disks list -f json`): -# import needs all three; nfs needs only EXISTING_DISK_ID. -# A static PV is created with reclaimPolicy=Retain (your disk is never deleted); -# PVC_SIZE becomes the PV capacity; STORAGE_CLASS is ignored. -EXISTING_DISK_ID= # crusoe disk `id` (nfs path = /volumes/; CSI volumeHandle) -EXISTING_DISK_NAME= # crusoe disk `name` -> csi.crusoe.ai/disk-name (import only) -EXISTING_DISK_SERIAL= # crusoe disk `serial_number` (import only) +# --- Import existing disk (DEST_MODE=import) --- +EXISTING_DISK_ID= +EXISTING_DISK_NAME= +EXISTING_DISK_SERIAL= EXISTING_DISK_FSTYPE=ext4 -# NFS-mode endpoint (the VAST data path on CMK): +# --- NFS mode (DEST_MODE=nfs) --- NFS_SERVER=nfs.crusoecloudcompute.com -NFS_EXPORT_PATH= # blank => /volumes/ +NFS_EXPORT_PATH= NFS_MOUNT_OPTIONS=vers=3,nconnect=16,spread_reads,spread_writes,remoteports=dns -# --- Fleet sizing --- -# TARGET_GBPS is the single sizing knob — set to your target aggregate GB/s. -TARGET_GBPS=30 -NUM_NODES=4 # nodes to spread across -PODS_PER_NODE=8 # workers per node (spread evenly). More pods = - # smaller shards = shorter straggler tail. Raise - # for shorter tails; if each pod uses many - # streams, watch node memory. -NUM_PODS= # absolute total override; default = NODES*PER_NODE +# --- Sizing --- +TARGET_GBPS=30.0 +NUM_NODES=4 +PODS_PER_NODE=8 +# NUM_PODS= # absolute override INSTANCE_CLASS=s2a - -# Node hardware (defaults = s2a.80x). Set for other SKUs/clouds so the sizing -# model uses the right CPU split, NIC line rate, and BDP. NODE_VCPU=80 NODE_RAM_GIB=676 NODE_NIC_GBPS=200 - -# Path RTT (ms) source<->destination region. Drives the BDP / concurrency math. -RTT_MS=150 -# Conservative per-stream throughput assumption (Mbps) for an untuned -# intercontinental TCP stream. Lower => more streams provisioned. Tune via sweep. -PER_STREAM_MBPS=250 -# Safety multiplier on the BDP-derived stream count. +RTT_MS=150.0 +PER_STREAM_MBPS=250.0 STREAM_SAFETY=1.5 -# --- rclone concurrency (blank => auto-derive per-pod from the BDP sizing) --- -# Leave TRANSFERS/STREAMS blank to size from TARGET_GBPS + RTT. To pin them, the -# Crusoe AWS-S3 reference used transfers=40, streams=40, no --checkers (strong -# throughput + low latency on large files): -# RCLONE_TRANSFERS=40 ; RCLONE_MULTI_THREAD_STREAMS=40 -RCLONE_TRANSFERS= # parallel files per pod (blank = auto) -RCLONE_MULTI_THREAD_STREAMS= # ranged-GET streams per file >cutoff (blank = auto) +# --- rclone (blank = auto-derive from sizing) --- +# RCLONE_TRANSFERS= +# RCLONE_MULTI_THREAD_STREAMS= RCLONE_MULTI_THREAD_CUTOFF=256M -RCLONE_MULTI_THREAD_CHUNK_SIZE=32M # smaller chunk = shorter per-file straggler tail -RCLONE_CHECKERS= # blank => omit --checkers (rclone default; with - # --no-traverse there's little to check + it - # avoids extra HEAD latency) +RCLONE_MULTI_THREAD_CHUNK_SIZE=32M +# RCLONE_CHECKERS= RCLONE_BUFFER_SIZE=32M RCLONE_S3_CHUNK_SIZE=64M -RCLONE_EXTRA_FLAGS= # e.g. "--low-level-retries 20 --timeout 120s" +# RCLONE_EXTRA_FLAGS= -# --- Worker pod resource requests (blank => auto-derived; the auto memory -# estimate is conservative for high stream counts, which can over-restrict how -# many pods schedule — set an explicit value when running 40/40). Requests -# affect scheduling only; with no limits set, actual use can exceed the request. -WORKER_CPU_REQUEST= -WORKER_MEM_REQUEST=16Gi +# --- Worker resources (blank = auto-derive) --- +# WORKER_CPU_REQUEST= +# WORKER_MEM_REQUEST= -# --- K8s plumbing --- +# --- K8s --- NAMESPACE=default -SECRET_NAME=cmk-data-transfer-oci +SECRET_NAME=cmk-data-transfer-s3 RCLONE_IMAGE=rclone/rclone:latest -KUBECONFIG= +# KUBECONFIG= diff --git a/cmk-data-transfer/.gitignore b/cmk-data-transfer/.gitignore index dc43630..5cce0b7 100644 --- a/cmk-data-transfer/.gitignore +++ b/cmk-data-transfer/.gitignore @@ -37,3 +37,4 @@ venv/ .vscode/ *.swp .claude/ +.superpowers/ diff --git a/cmk-data-transfer/Makefile b/cmk-data-transfer/Makefile index 3b877d3..8e82e65 100644 --- a/cmk-data-transfer/Makefile +++ b/cmk-data-transfer/Makefile @@ -17,7 +17,7 @@ sizing: ## Print the BDP/target sizing plan (no cluster access) dry-run: ## Render manifests + run preflight checks, launch nothing $(PY) -m orchestrator.run --dry-run -preflight: ## VAST write-ceiling test (fio, one pod per node) — safe, no egress +preflight: ## Disk write-ceiling test (fio, one pod per node) — safe, no egress $(PY) preflight/run_fio.py fio-bench: ## Disk read+write benchmark: fio on every s2a node -> bench/results/ @@ -45,7 +45,7 @@ clean: ## Delete worker + master + fio pods (keep PVC/Secret/data) -kubectl -n $(NAMESPACE) delete pod cmk-data-transfer-master --ignore-not-found clean-all: clean ## Also delete the Secret (keeps the PVC and downloaded data) - -kubectl -n $(NAMESPACE) delete secret cmk-data-transfer-oci --ignore-not-found + -kubectl -n $(NAMESPACE) delete secret cmk-data-transfer-s3 --ignore-not-found @echo "PVC and downloaded data left intact. Delete the PVC manually if desired." # --- defaults for sweep knobs (override on CLI) --- diff --git a/cmk-data-transfer/README.md b/cmk-data-transfer/README.md index 50204d2..773d612 100644 --- a/cmk-data-transfer/README.md +++ b/cmk-data-transfer/README.md @@ -1,11 +1,13 @@ # CMK Cross-Region Object Storage Data Transfer -Parallel-pull a dataset from **any S3-compatible object store** to a **VAST-backed -RWX shared disk** on **Crusoe Managed Kubernetes (CMK)**, tuned to saturate worker -hosts across a high-latency path. **OCI Object Storage is the worked example**, but -the source backend is rclone's generic `provider = Other`, so AWS S3, MinIO/Ceph, -GCS (S3 interop), Cloudflare R2, Backblaze B2, etc. work too — see -[Other S3-compatible sources](#other-s3-compatible-sources). +Parallel-pull a dataset from **any S3-compatible object store** to a **Crusoe shared filesystem +RWX shared disk** or **S3-compatible destination** on **Crusoe Managed Kubernetes +(CMK)**, tuned to saturate worker hosts across a high-latency path. **AWS S3 is the +worked example**, but the source backend is rclone's generic `provider = Other`, so +OCI Object Storage, MinIO/Ceph, GCS (S3 interop), Cloudflare R2, Backblaze B2, +etc. work too — see [Other S3-compatible sources](#other-s3-compatible-sources). +Set `DEST_TYPE=s3` to write directly to Crusoe S3 (or any S3-compatible +destination) instead of a shared disk. A **master pod** lists the source and writes balanced shard manifests to the shared disk; **N worker pods** each `rclone copy` one shard in parallel. It follows Crusoe's @@ -18,17 +20,36 @@ concurrency**. ## Quick Start ```bash -cp .env.example .env # fill in creds, namespace, region, bucket, destination disk +cp .env.example .env # fill in creds, namespace, bucket, and destination make sizing # (optional) print the concurrency plan — no cluster access make dry-run # render manifests + preflight checks (launches nothing) make preflight # (optional) fio write-ceiling test (safe, no egress) make run # full pipeline; PROMPTS before the large transfer ``` -> **Where does the data land?** The shared disk mounts at **`/data`** in every pod, -> and objects copy to **`DEST_PATH`** (default `/data/dataset`) — set it in `.env`. -> **Using an existing disk?** Set `DEST_MODE=import` (or `nfs`) + the disk id, then -> point `DEST_PATH` at where on that disk you want the files. See +**Disk destination** (default — writes to a Crusoe shared filesystem RWX shared disk): + +```bash +# In .env: +DEST_TYPE=disk # or omit (disk is the default) +DEST_PATH=/data/dataset +``` + +**S3 destination** (copies directly to Crusoe S3 or any S3-compatible store): + +```bash +# In .env: +DEST_TYPE=s3 +DEST_S3_ENDPOINT=https://s3.crusoe.ai +DEST_S3_BUCKET=my-dest-bucket +DEST_S3_ACCESS_KEY_ID=... +DEST_S3_SECRET_ACCESS_KEY=... +``` + +> **Disk mode — where does the data land?** The shared disk mounts at **`/data`** in +> every pod, and objects copy to **`DEST_PATH`** (default `/data/dataset`) — set it in +> `.env`. **Using an existing disk?** Set `DEST_MODE=import` (or `nfs`) + the disk id, +> then point `DEST_PATH` at where on that disk you want the files. See > [Destination modes](#destination-modes-dest_mode). --- @@ -41,7 +62,7 @@ make run # full pipeline; PROMPTS before the large transfer │ list (via master) → bin-pack shards locally → push to shared disk ▼ ┌──────────────────────────── Crusoe Managed Kubernetes ───────────────────────┐ -│ Secret(rclone.conf, RO) PVC: shared disk (VAST, RWX) │ +│ Secret(rclone.conf, RO) PVC: shared disk (Crusoe shared filesystem, RWX) │ │ ▼ ▼ │ │ ┌─────────────┐ rclone lsf ┌─────────────────────────────────────┐ │ │ │ master pod │ ───────────────► │ shared disk /data │ │ @@ -56,8 +77,14 @@ make run # full pipeline; PROMPTS before the large transfer └─────────┼──────────────┼────────────────────┼──────────────┼──────────────────┘ ▼ ▼ ▼ ▼ hundreds of parallel ┌──────────────────────────────────────────────────┐ ranged-GET streams - │ S3-compatible object store (e.g. OCI) │ high RTT (intercontinental) + │ S3-compatible source (AWS S3, OCI, R2, …) │ high RTT (intercontinental) └──────────────────────────────────────────────────┘ + DEST_TYPE=s3: workers + also write to an + S3-compatible destination + (e.g. Crusoe S3) using a + small 1 TiB coordination + PVC (shards + logs only) ``` The fleet runs **`PODS_PER_NODE` independent rclone processes per node** — multiple @@ -128,34 +155,34 @@ All inputs come from `.env` (copied from `.env.example`), process env, or CLI fl | Variable | Meaning | |---|---| -| `OCI_ACCESS_KEY_ID` / `OCI_SECRET_ACCESS_KEY` | S3-compat access/secret key pair (OCI: "Customer Secret Key") | -| `OCI_NAMESPACE` / `OCI_REGION` | OCI Object Storage namespace + region (auto-build the endpoint) | -| `OCI_BUCKET` / `OCI_PREFIX` | source bucket and optional prefix | -| `OCI_ENDPOINT` | explicit S3 endpoint (for non-OCI stores; blank = auto from namespace+region) | +| `SRC_S3_ACCESS_KEY_ID` / `SRC_S3_SECRET_ACCESS_KEY` | S3-compat access/secret key pair for the source | +| `SRC_S3_ENDPOINT` | source S3 endpoint URL (e.g. `https://s3.us-east-1.amazonaws.com`) | +| `SRC_S3_REGION` | source region (optional, depends on provider) | +| `SRC_S3_BUCKET` / `SRC_S3_PREFIX` | source bucket and optional prefix | +| `DEST_TYPE` | `disk` (default) or `s3` — destination type | +| `DEST_S3_ACCESS_KEY_ID` / `DEST_S3_SECRET_ACCESS_KEY` | S3-compat credentials for the destination (when `DEST_TYPE=s3`) | +| `DEST_S3_ENDPOINT` / `DEST_S3_REGION` | destination S3 endpoint + region (when `DEST_TYPE=s3`) | +| `DEST_S3_BUCKET` / `DEST_S3_PREFIX` | destination bucket and optional prefix (when `DEST_TYPE=s3`) | | `TARGET_GBPS` | **the single sizing knob** (default 30) | | `NUM_NODES` / `PODS_PER_NODE` | fleet size; total = nodes × pods/node (`NUM_PODS` forces an absolute total) | | `INSTANCE_CLASS` | node class for the nodeSelector (default `s2a`) | | `NODE_VCPU` / `NODE_RAM_GIB` / `NODE_NIC_GBPS` | per-node hardware for the sizing model (defaults = s2a.80x) | | `RTT_MS` / `PER_STREAM_MBPS` / `STREAM_SAFETY` | BDP model inputs | -| `DEST_PATH` | **where objects are saved** — folder under the `/data` mount (default `/data/dataset`) | -| `DEST_MODE` | `dynamic` / `import` / `nfs` destination (see below) | -| `STORAGE_CLASS` / `PVC_NAME` / `PVC_SIZE` | destination shared disk | +| `DEST_PATH` | **where objects are saved** — folder under the `/data` mount (default `/data/dataset`, disk mode only) | +| `DEST_MODE` | `dynamic` / `import` / `nfs` destination (disk mode only; see below) | +| `STORAGE_CLASS` / `PVC_NAME` / `PVC_SIZE` | destination shared disk (disk mode only) | | `EXISTING_DISK_ID` / `_NAME` / `_SERIAL` / `NFS_SERVER` | bind an existing disk (import/nfs modes) | | `RCLONE_*` / `WORKER_MEM_REQUEST` | per-flag / resource overrides (blank = auto-derive) | -> **Region note:** use the **OCI** region slug where the *bucket* lives (not your -> destination region). Preflight warns if it doesn't look like an OCI slug. - ### Other S3-compatible sources The source backend is rclone `provider = Other`, so any S3-compatible store works. -The `OCI_*` names are just labels — for a non-OCI store, set `OCI_ENDPOINT` to its S3 -endpoint and use its access/secret key + bucket: +Set `SRC_S3_ENDPOINT` to its S3 endpoint and use its access/secret key + bucket: -| Source | `OCI_ENDPOINT` | +| Source | `SRC_S3_ENDPOINT` | |---|---| -| OCI Object Storage | *(auto from namespace + region)* | | AWS S3 | `https://s3..amazonaws.com` (or blank with a real AWS key) | +| OCI Object Storage | `https://.compat.objectstorage..oraclecloud.com` | | MinIO / Ceph RGW | `https://:` | | Google Cloud Storage | `https://storage.googleapis.com` (S3 interop + HMAC key) | | Cloudflare R2 | `https://.r2.cloudflarestorage.com` | @@ -176,14 +203,16 @@ lands in the repo, in pod args, or in shell history. `.gitignore` blocks `.env`, 1. **Sizing + preflight** — prints the concurrency plan; verifies `kubectl`, enough schedulable nodes (`INSTANCE_CLASS`), and the StorageClass (creates it if - absent). -2. **Secret** — builds `rclone.conf`, applies it in-cluster. -3. **PVC + master pod** — applies the RWX claim and the master. + absent, disk mode only). +2. **Secret** — builds `rclone.conf` (source + optional dest remote), applies it in-cluster. +3. **PVC + master pod** — provisions the coordination PVC (large RWX disk for + `DEST_TYPE=disk`; small 1 TiB for `DEST_TYPE=s3`) and launches the master pod. 4. **List + shard** — `rclone lsf` in the master → pull the listing → bin-pack (LPT) into N balanced shard files → push to `/data/shards/`. 5. **Confirm** — prompts before the large transfer (skip with `--yes`). 6. **Launch** — `PODS_PER_NODE` workers per node (pinned), each `rclone copy - --files-from shard-i.txt --no-traverse …`. + --files-from shard-i.txt --no-traverse …` (to disk or S3 dest depending on + `DEST_TYPE`). 7. **Monitor → teardown** — polls to completion, then deletes worker + master pods (keeps PVC, Secret, data) unless `--keep`. @@ -198,7 +227,7 @@ only fetches what's missing. |---|---|---|---| | `dynamic` (default) | provisions a new disk via the fs CSI driver | — | greenfield | | `import` | binds an **existing** disk via a CSI **static PV** | disk `id` + `name` + `serial` | migrate into an existing disk, CSI healthy | -| `nfs` | binds an **existing** disk via an **in-tree NFS PV** to the VAST DNS endpoint (bypasses CSI) | disk `id` (+ `NFS_SERVER`) | when a CSI mount times out on an unroutable fallback IP | +| `nfs` | binds an **existing** disk via an **in-tree NFS PV** to the Crusoe shared filesystem DNS endpoint (bypasses CSI) | disk `id` (+ `NFS_SERVER`) | when a CSI mount times out on an unroutable fallback IP | Find the disk with `crusoe storage disks list -f json` (pick the `shared-volume` in your region): `.id`, `.name`, `.serial_number`. All modes use `reclaimPolicy: Retain`, @@ -217,13 +246,45 @@ Because `rclone copy` is idempotent, `DEST_PATH` can point at a folder already h part of the dataset — only missing/changed objects are pulled. > **`nfs` mode** exists because the fs CSI driver can fall back to a fixed IP that may -> be unroutable from your nodepool (mounts hang). The VAST DNS endpoint +> be unroutable from your nodepool (mounts hang). The Crusoe shared filesystem DNS endpoint > (`nfs.crusoecloudcompute.com`, `remoteports=dns`) mounts cleanly; `nfs` mode creates > an in-tree NFS PV straight to it (see `k8s/nfs-pv.yaml`). If a CSI mount hangs, > prefer `nfs`. Export path defaults to `/volumes/`. --- +## S3-to-S3 mode (`DEST_TYPE=s3`) + +Set `DEST_TYPE=s3` to copy directly from a source S3-compatible store to a destination +S3-compatible store (e.g. Crusoe S3), bypassing the shared disk entirely. Workers run +`rclone copy src:bucket dest:bucket` instead of writing to the mounted PVC. + +A **small 1 TiB coordination PVC** is still provisioned to hold shard manifests and +worker logs — no large data disk is needed. + +```bash +SRC_S3_ACCESS_KEY_ID= \ +SRC_S3_SECRET_ACCESS_KEY= \ +SRC_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com \ +SRC_S3_BUCKET=my-source-bucket \ +DEST_TYPE=s3 \ +DEST_S3_ACCESS_KEY_ID= \ +DEST_S3_SECRET_ACCESS_KEY= \ +DEST_S3_ENDPOINT=https://s3.crusoe.ai \ +DEST_S3_BUCKET=my-dest-bucket \ +make run +``` + +Or place all variables in `.env` and just run `make run`. The `DEST_S3_*` credentials +are kept in the same Kubernetes Secret as the source credentials, assembled into a +single `rclone.conf` with `[src]` and `[dest]` remotes. + +> **Note:** `DEST_MODE`, `STORAGE_CLASS`, `PVC_SIZE`, and disk-specific settings are +> ignored when `DEST_TYPE=s3`. Use `RCLONE_EXTRA_FLAGS` as an escape hatch for +> destination-specific rclone options. + +--- + ## Benchmarking - **`make preflight`** — fio write-ceiling, one pod per node, no egress. If the @@ -289,26 +350,6 @@ kubectl get po -l app=cmk-data-transfer-worker --field-selector=status.phase=Run --- -## Alternative: native OCI backend - -Instead of the S3-compat path you can use rclone's native `oracleobjectstorage` -backend (OCI IAM auth, no access/secret key): - -```ini -[oci-native] -type = oracleobjectstorage -namespace = -region = -provider = user_principal_auth # or instance_principal_auth / resource_principal_auth -config_file = /root/.oci/config -config_profile = DEFAULT -``` - -Mount your `~/.oci` config + key into the pods and point the remote at `oci-native:`. -The S3-compat path is the default because it matches the credential most users hold. - ---- - ## Repo layout ``` diff --git a/cmk-data-transfer/bench/fio/fio-bench-job.yaml b/cmk-data-transfer/bench/fio/fio-bench-job.yaml index e3d37db..aedafde 100644 --- a/cmk-data-transfer/bench/fio/fio-bench-job.yaml +++ b/cmk-data-transfer/bench/fio/fio-bench-job.yaml @@ -1,7 +1,7 @@ -# Disk benchmark: one fio pod per s2a node, against the shared VAST PVC. +# Disk benchmark: one fio pod per s2a node, against the shared filesystem PVC. # # Runs four stonewalled (sequential) fio profiles against a single per-node file -# so reads measure data actually on VAST (direct=1 bypasses the page cache): +# so reads measure data actually on disk (direct=1 bypasses the page cache): # seqwrite (1M) -> sequential write bandwidth # seqread (1M) -> sequential read bandwidth (re-reads the written file) # randwrite (4k) -> random write IOPS diff --git a/cmk-data-transfer/bench/fio/run_fio_bench.py b/cmk-data-transfer/bench/fio/run_fio_bench.py index 079180a..3a1ba2d 100644 --- a/cmk-data-transfer/bench/fio/run_fio_bench.py +++ b/cmk-data-transfer/bench/fio/run_fio_bench.py @@ -10,8 +10,8 @@ [--size 10G] [--jobs 4] [--direct 1] [--instance-class s2a] [--namespace default] [--image alpine:3.20] [--timeout 1200] -Safe: writes only under /data/_fiobench// on the PVC and cleans up. No OCI -egress (VAST I/O is in-region). +Safe: writes only under /data/_fiobench// on the PVC and cleans up. No +egress (disk I/O is in-region). """ from __future__ import annotations diff --git a/cmk-data-transfer/bench/oci_experiment.py b/cmk-data-transfer/bench/oci_experiment.py index f262520..af9cbfd 100644 --- a/cmk-data-transfer/bench/oci_experiment.py +++ b/cmk-data-transfer/bench/oci_experiment.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""OCI -> VAST transfer experiment: measure sustained pull rate at a given +"""S3 transfer experiment: measure sustained pull rate at a given node/pod concurrency, writing to a DISTINCT destination subfolder per run. Drives the orchestrator (secret -> PVC/master -> list -> shard -> workers) for a @@ -8,7 +8,7 @@ - bytes transferred + wall-clock elapsed - average GB/s (= bytes / elapsed) <- the headline "how fast did we pull" - peak windowed GB/s (steady-state) - - per-node GB/s, and any OCI throttling (429/503 SlowDown) seen in logs + - per-node GB/s, and any S3 throttling (429/503 SlowDown) seen in logs Time-series + summary saved to bench/results/oci-exp-