diff --git a/.githooks/pre-push b/.githooks/pre-push index 8b3bea48..06f99081 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -5,7 +5,7 @@ set -eo pipefail REPO_ROOT="$(git rev-parse --show-toplevel)" LOG_FILE="${REPO_ROOT}/.githooks/pre-push.log" -echo "[pre-push] Running full verification suite (output -> ${LOG_FILE})..." >&2 +echo "[pre-push] Running fast verification suite (output -> ${LOG_FILE})..." >&2 if "${REPO_ROOT}/scripts/test-before-push.sh" > "${LOG_FILE}" 2>&1; then echo "[pre-push] PASS" >&2 exit 0 diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml deleted file mode 100644 index 5fbf29ae..00000000 --- a/.github/workflows/quality.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Quality - -on: - pull_request: - push: - branches: - - main - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - -permissions: - contents: read - -jobs: - fmt: - name: Format Check - runs-on: ubuntu-24.04 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly-2026-04-21 - components: rustfmt - - - name: Format check - run: cargo fmt --all -- --check - - rust-quality: - name: Rust Quality - runs-on: ubuntu-24.04 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install build dependencies - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: protobuf-compiler pkg-config libssl-dev - version: quality-ubuntu24-v1 - - - name: Set up Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly-2026-04-21 - components: rustfmt, clippy - - - name: Cache Rust build artifacts - uses: swatinem/rust-cache@v2 - with: - shared-key: quality-ubuntu24 - cache-on-failure: true - - - name: Clippy - run: cargo clippy --all-targets --all-features -- -D warnings - - - name: Test - run: cargo test --tests -- --test-threads=1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..af1f2890 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,120 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + version: + description: "EloqKV version" + required: false + default: "1.2.2" + workers: + description: "Stress test workers" + required: false + default: "32" + duration: + description: "Stress duration (seconds)" + required: false + default: "60" + repeat: + description: "Per-command repeat factor" + required: false + default: "10" + inflight: + description: "Concurrent execution slots per worker" + required: false + default: "4" + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + +permissions: + contents: read + +jobs: + test: + name: Test + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - name: Install build dependencies + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: protobuf-compiler pkg-config libssl-dev + version: test-ubuntu24-v1 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-04-21 + components: rustfmt, clippy + + - name: Cache Rust build artifacts + uses: swatinem/rust-cache@v2 + with: + shared-key: test-ubuntu24 + cache-on-failure: true + + - name: Format check + run: cargo fmt --all -- --check + + - name: Clippy + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Unit tests + run: cargo test --tests -- --test-threads=1 + + - name: Build eloqctl + run: | + cargo build -p cluster_mgr + mkdir -p ~/.eloqctl/bin + cp target/debug/cluster_mgr ~/.eloqctl/bin/cluster_mgr + chmod 755 ~/.eloqctl/bin/cluster_mgr + rm -f ~/.eloqctl/config + ln -s "$PWD/src/cluster_mgr/config" ~/.eloqctl/config + + - name: Run E2E test + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + workers="${{ github.event.inputs.workers || '32' }}" + duration="${{ github.event.inputs.duration || '60' }}" + repeat="${{ github.event.inputs.repeat || '10' }}" + inflight="${{ github.event.inputs.inflight || '4' }}" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + workers="16" + duration="60" + repeat="10" + inflight="4" + else + workers="32" + duration="60" + repeat="20" + inflight="4" + fi + + export DOCKER_E2E_DIR="$PWD/tests/docker_ha" + export ELOQCTL_HOME="$HOME/.eloqctl" + export ELOQCTL="$HOME/.eloqctl/bin/cluster_mgr" + export ELOQKV_VERSION="${{ github.event.inputs.version || '1.2.2' }}" + export DURATION_SECONDS="${duration}" + export WORKERS="${workers}" + export INFLIGHT="${inflight}" + export REPEAT="${repeat}" + export SKIP_DEPS=1 + export STEPS="launch,py-stress,go-stress,ts-stress,remove" + echo "Running E2E with WORKERS=${WORKERS} INFLIGHT=${INFLIGHT} DURATION_SECONDS=${DURATION_SECONDS} REPEAT=${REPEAT}" + bash tests/e2e/cmd_stress_test.sh + + - name: Cleanup Docker + if: always() + run: | + docker compose -f tests/docker_ha/docker-compose.yaml down -v --remove-orphans 2>/dev/null || true + docker system prune -af --volumes 2>/dev/null || true diff --git a/.gitignore b/.gitignore index e263bbc9..2856441b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ src/log/ log/ .monograph_waiter/ .bin/ +.tmp-eloqctl/ tests/ssh_keys/test_rsa tests/**/.eloqctl/ tests/**/topology.generated.yaml @@ -46,3 +47,6 @@ tests/**/id_ed25519 tests/**/id_ed25519.pub tests/**/authorized_keys tests/**/exported.yaml +tests/**/node_modules/ +tests/**/__pycache__/ +tests/**/go-cluster-tls-stress diff --git a/Cargo.lock b/Cargo.lock index c03a50d1..5003cf64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1086,7 +1086,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cluster_mgr" -version = "1.6.4" +version = "1.6.6" dependencies = [ "anyhow", "async-trait", @@ -1100,6 +1100,7 @@ dependencies = [ "crossbeam-channel", "dirs", "dyn-clone", + "flate2", "futures", "futures-async-stream", "humantime", @@ -1125,10 +1126,12 @@ dependencies = [ "serde_json", "serde_with", "serde_yaml", + "sha2", "sqlx", "strum_macros", "sysinfo 0.30.12", "tabled", + "tar", "thiserror", "tokio", "tokio-postgres", @@ -1529,7 +1532,7 @@ dependencies = [ [[package]] name = "devtools" -version = "1.6.4" +version = "1.6.6" dependencies = [ "anyhow", "async-trait", @@ -1831,6 +1834,16 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.4" @@ -3692,7 +3705,7 @@ dependencies = [ [[package]] name = "rest_api" -version = "1.6.4" +version = "1.6.6" dependencies = [ "actix-server", "actix-web", @@ -4657,6 +4670,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -5588,6 +5612,17 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "xattr" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/doc/cluster_mgr.md b/doc/cluster_mgr.md index fffb37bd..8b6f4b5c 100644 --- a/doc/cluster_mgr.md +++ b/doc/cluster_mgr.md @@ -49,7 +49,7 @@ eloqctl remove eloqkv-cluster --force | `connect ` | Print an EloqKV client command for the cluster. | | `plan ` | Preview supported changes without mutating local state or remote hosts. | | `apply ` | Execute the same plan shown by `plan`, gated by live critical-service health and verified afterward. | -| `update ` | Rolling version update for an existing cluster. | +| `update ` | Rolling version update for an existing cluster. Version lookup and tarball selection come from GitHub Releases for `eloqdata/eloqkv`. `--download-only` resolves and caches the required tarballs without touching remote hosts. `--monitor` updates only one monitor component. | | `update-conf ` | Apply selected config fields and optionally restart tx nodes. | | `scale ` | Add or remove EloqKV tx/standby nodes. Duplicate add/remove requests are no-ops. | | `scalelog ` | Add or remove log service nodes. Duplicate add/remove requests are no-ops. | @@ -59,7 +59,7 @@ eloqctl remove eloqkv-cluster --force | `backup ...` | Create, list, remove, restore, and dump backups. | | `export [--output file]` | Write the saved launch-compatible topology YAML. | | `list` | List locally registered clusters. | -| `versions` | List available EloqKV versions. | +| `versions` | List available EloqKV versions by scanning GitHub Release assets for `eloqdata/eloqkv`. | | `completion ` | Generate shell completion scripts. | | `upgrade` | Run local SQLite schema upgrades. | | `remove [--force]` | Remove local cluster metadata and perform best-effort remote cleanup. | diff --git a/doc/upgrade.md b/doc/upgrade.md index d625c695..2ef466b0 100644 --- a/doc/upgrade.md +++ b/doc/upgrade.md @@ -32,6 +32,38 @@ Upgrade an existing EloqKV cluster to a specific version: eloqctl update ``` +`eloqctl` now resolves available versions and EloqKV tarballs from GitHub Releases for `eloqdata/eloqkv`. For example, asset `eloqkv-1.2.2-rocks_s3-ubuntu24-amd64.tar.gz` is treated as version `1.2.2` for store `rocks_s3`. + +Download the required tarballs into the local cache without applying the update: + +```sh +eloqctl update --download-only +``` + +`--download-only` still resolves the version against the saved cluster topology, so provide the existing cluster name and the target version exactly as you would for a real upgrade. It downloads the matching EloqKV release tarballs into `$ELOQCTL_HOME/download/...` and exits without changing remote files, processes, or cluster metadata. + +Typical workflow: + +```sh +eloqctl versions +eloqctl update 1.2.2 --download-only +eloqctl update 1.2.2 +``` + +Update only one monitor component without touching the EloqKV cluster itself: + +```sh +eloqctl update --monitor grafana +``` + +Override the monitor tarball URL for a monitor-only update: + +```sh +eloqctl update --monitor grafana --monitor-url https://dl.grafana.com/oss/release/grafana-11.0.0.linux-amd64.tar.gz +``` + +GitHub-hosted EloqKV tarballs are now validated with the asset `sha256` digest when available. Cached files are reused only when the digest still matches, so a re-published asset under the same tag can be refreshed correctly. + Upgrade to the latest available version: ```sh diff --git a/scripts/test-before-push.sh b/scripts/test-before-push.sh index 45d3d6bc..4b4fd854 100755 --- a/scripts/test-before-push.sh +++ b/scripts/test-before-push.sh @@ -7,18 +7,15 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" cd "${REPO_ROOT}" -echo "[1/5] Install dev eloqctl" +echo "[1/4] Install dev eloqctl" "${REPO_ROOT}/scripts/install-dev.sh" -echo "[2/5] Check formatting" +echo "[2/4] Check formatting" cargo fmt --all -- --check -echo "[3/5] Check and lint" +echo "[3/4] Check and lint" cargo check -p cluster_mgr cargo clippy --all-targets --all-features -- -D warnings -echo "[4/5] Run Docker E2E suite" -bash tests/e2e/test.sh - -echo "[5/5] Done" -echo "PASS: pre-push test suite completed" +echo "[4/4] Done" +echo "PASS: pre-push checks completed (full E2E runs in GitHub Actions)" diff --git a/src/cluster_mgr/Cargo.toml b/src/cluster_mgr/Cargo.toml index df5761e4..23d2f047 100644 --- a/src/cluster_mgr/Cargo.toml +++ b/src/cluster_mgr/Cargo.toml @@ -20,6 +20,7 @@ futures = { version = "0.3", default-features = false, features = [ "alloc", "thread-pool", ] } +flate2 = "1" futures-async-stream = "0.2" indexmap = "2.1.0" indicatif = "0.17" @@ -36,6 +37,8 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" serde_with = "1.6.1" +sha2 = "0.10" +tar = "0.4" sqlx = { version = "0.7.3", features = [ "runtime-tokio-native-tls", "sqlite", diff --git a/src/cluster_mgr/config/alert.rules b/src/cluster_mgr/config/alert.rules index 5282151d..ca71deed 100644 --- a/src/cluster_mgr/config/alert.rules +++ b/src/cluster_mgr/config/alert.rules @@ -114,4 +114,22 @@ groups: annotations: summary: "Leader changes frequently" description: "More than three leadership changes were seen within 10 minutes for node group {{ $labels.ng_id }}" - \ No newline at end of file + + - alert: eloqkv_instance_ops_too_low + expr: sum by (instance) (rate(redis_command_aggregated_total[1m])) < 100000 + for: 1m + labels: + severity: critical + annotations: + summary: "EloqKV instance OPS too low" + description: "EloqKV instance {{ $labels.instance }} OPS has been below 10000 for 1 minutes. Current value: {{ printf \"%.2f\" $value }}" + + - alert: eloqkv_active_connections_too_high + expr: sum by (instance) (redis_connection_count) > 100 + for: 1m + labels: + severity: warning + annotations: + summary: "EloqKV active connections too high" + description: "EloqKV instance {{ $labels.instance }} active connections have exceeded 1000 for 1 minute. Current value: {{ printf \"%.0f\" $value }}" + diff --git a/src/cluster_mgr/config/mono_prometheus.yaml b/src/cluster_mgr/config/mono_prometheus.yaml index aa3f184a..94b41efd 100644 --- a/src/cluster_mgr/config/mono_prometheus.yaml +++ b/src/cluster_mgr/config/mono_prometheus.yaml @@ -7,10 +7,11 @@ rule_files: - "alert.rules" # Path relative to Prometheus CWD or absolute path # A scrape configuration containing exactly one endpoint to scrape. +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - '127.0.0.1:9093' + scrape_configs: -#alerting: -# alertmanagers: -# - scheme: http -# static_configs: -# - targets: -# - 'alertmanager:9093' \ No newline at end of file diff --git a/src/cluster_mgr/src/cli/cmd_base.rs b/src/cluster_mgr/src/cli/cmd_base.rs index 759d693d..b39b6908 100644 --- a/src/cluster_mgr/src/cli/cmd_base.rs +++ b/src/cluster_mgr/src/cli/cmd_base.rs @@ -2,18 +2,28 @@ use crate::cli::reconcile::{ ObservedCluster, ObservedServiceStatus, ReconcileAction, ReconcilePlan, }; use crate::cli::task::backup_utils::split_manifests; +use crate::cli::task::download_task::DownloadTask; use crate::cli::task::group::Config; -use crate::cli::task::task_base::{set_verbose_task_output, TaskMgr, TaskResultPair}; +use crate::cli::task::monitor_ctl_task::MonitorCtlTask; +use crate::cli::task::task_base::{ + set_verbose_task_output, TaskExecutionContext, TaskMgr, TaskResultPair, +}; +use crate::cli::task::unpack_file_task::UnpackFileTask; +use crate::cli::task::upload::upload_task_builder::{upload_tasks, UploadTaskBuilderType}; use crate::cli::util::{cpu_arch, file_pg_bar, os_id, os_major_version}; -use crate::cli::{upload_dir, SubCommand, HOME_DIR}; +use crate::cli::{upload_dir, SubCommand, UpdateMonitorComponent, HOME_DIR}; use crate::cli::{BackupCommand, ProxyCommand}; use crate::config::config_base::{DeployConfig, VersionRow}; -use crate::config::deployment::{Deployment, Product}; +use crate::config::config_base::{GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, PROMETHEUS_FILE_KEY}; +use crate::config::deployment::{version_digits, Deployment, Product}; use crate::config::proxy_config_base::ProxyConfig; use crate::config::storage_service_config::{ DataStoreServiceBackend, DataStoreServiceMode, RocksDB, RocksLocal, StorageService, }; use crate::config::{DeploymentPackage, StorageProvider, CDN, CONFIG_PATH_DIR, UPLOAD_PATH_DIR}; +use crate::github_release::{ + fetch_eloqkv_releases, find_eloqkv_asset, find_product_asset, list_versions_from_releases, +}; use crate::state::proxy_operation::{ProxyEntity, ProxyOperation}; use crate::state::state_base::{QueryCondition, StateOperation}; use crate::state::state_mgr::{StateMgr, PROXY_STATE, STATE_MGR}; @@ -29,10 +39,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, LazyLock}; use std::time::Duration; use std::{env, fs}; -use tokio::sync::OnceCell; -use tokio_postgres::config::SslMode; -use tokio_postgres::NoTls; -use tracing::{error, info, warn}; +use tracing::{info, warn}; #[derive(tabled::Tabled)] struct StatusSummaryRow { @@ -63,7 +70,6 @@ pub static HTTP_INTERNAL: LazyLock = LazyLock::new(|| { pub struct CmdExecutor { task_mgr: Arc, state_mgr: Arc, - pg_client: OnceCell, pub home: PathBuf, } @@ -163,7 +169,6 @@ impl CmdExecutor { Self { task_mgr: Arc::new(TaskMgr::new()), state_mgr: Arc::new(STATE_MGR.clone()), - pg_client: OnceCell::new(), home, } } @@ -205,33 +210,6 @@ impl CmdExecutor { Ok(home) } - async fn pg_client(&self) -> Result<&tokio_postgres::Client> { - self.pg_client - .get_or_try_init(|| async { - let pg_password = env::var("ELOQCTL_PG_PASSWORD") - .unwrap_or_else(|_| "eloq_readonly123!".to_string()); - let (client, conn) = tokio_postgres::Config::new() - .user("readonly_user") - .password(pg_password) - .host("18.177.72.104") - .port(5432) - .dbname("eloq_release") - .ssl_mode(SslMode::Disable) - .connect(NoTls) - .await - .map_err(|e| anyhow!("connect postgres failed: {e}"))?; - // The connection object performs the actual communication with the database, - // so spawn it off to run on its own. - tokio::spawn(async move { - if let Err(e) = conn.await { - error!("PG connection error: {}", e); - } - }); - Ok(client) - }) - .await - } - pub fn task_mgr(&self) -> &Arc { &self.task_mgr } @@ -275,6 +253,55 @@ impl CmdExecutor { println!("{}", tabled::Table::new(rows)); } + fn shell_single_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\"'\"'")) + } + + fn redis_cli_connect_command( + config: &DeployConfig, + cli_password: Option<&str>, + ) -> Option { + let host_port = config.deployment.tx_service.tx_host_ports.first()?; + let parts: Vec<&str> = host_port.split(':').collect(); + let port = parts.get(1)?.parse::().ok()?; + let (host, port) = config.service_endpoint(parts[0], port); + + let mut command = format!("redis-cli -h {host} -p {port}"); + if let Some(password) = config.redis_password(cli_password.map(ToOwned::to_owned)) { + command.push_str(" -a "); + command.push_str(&Self::shell_single_quote(&password)); + } + Some(command) + } + + fn print_status_connect_hint(config: &DeployConfig, cli_password: Option<&str>) { + if let Some(command) = Self::redis_cli_connect_command(config, cli_password) { + println!("Connect with redis-cli:\n\t{command}"); + } + } + + fn print_status_output( + cmd: &SubCommand, + deploy_config: &DeployConfig, + task_results: &[TaskResultPair], + verbose: bool, + ) { + if let SubCommand::Status { + detail, password, .. + } = cmd + { + if !detail { + Self::print_status_summary(task_results); + Self::print_status_connect_hint(deploy_config, password.as_deref()); + if !verbose { + println!("Tip: use `--verbose` to show per-task execution details."); + } + } else { + Self::print_status_connect_hint(deploy_config, password.as_deref()); + } + } + } + async fn observe_cluster( &'static self, config: &DeployConfig, @@ -463,6 +490,488 @@ impl CmdExecutor { self.home.join("download") } + fn update_monitor_component_name(component: &UpdateMonitorComponent) -> &'static str { + match component { + UpdateMonitorComponent::Grafana => "grafana", + UpdateMonitorComponent::Prometheus => "prometheus", + UpdateMonitorComponent::NodeExporter => "node_exporter", + } + } + + fn configure_monitor_update( + config: &mut DeployConfig, + component: &UpdateMonitorComponent, + monitor_url: Option, + ) -> Result<()> { + let monitor = config.deployment.monitor.as_mut().ok_or_else(|| { + anyhow!( + "cluster '{}' has no monitor configuration", + config.deployment.cluster_name + ) + })?; + + match component { + UpdateMonitorComponent::Grafana => { + let grafana = monitor.grafana.as_mut().ok_or_else(|| { + anyhow!( + "cluster '{}' has no grafana configuration", + config.deployment.cluster_name + ) + })?; + if let Some(url) = monitor_url { + grafana.download_url = url; + } + } + UpdateMonitorComponent::Prometheus => { + let prometheus = monitor.prometheus.as_mut().ok_or_else(|| { + anyhow!( + "cluster '{}' has no prometheus configuration", + config.deployment.cluster_name + ) + })?; + if let Some(url) = monitor_url { + prometheus.download_url = url; + } + } + UpdateMonitorComponent::NodeExporter => { + let exporter = monitor.node_exporter.as_mut().ok_or_else(|| { + anyhow!( + "cluster '{}' has no node_exporter configuration", + config.deployment.cluster_name + ) + })?; + if let Some(url) = monitor_url { + exporter.url = url; + } + } + } + Ok(()) + } + + fn monitor_update_context( + &self, + cmd: &SubCommand, + config: &DeployConfig, + component: &UpdateMonitorComponent, + ) -> TaskExecutionContext { + let mut executable = indexmap::IndexMap::new(); + let mut barrier = vec![]; + + let download = DownloadTask::from_config(config).unwrap_or_default(); + if !download.is_empty() { + barrier.push(download.len()); + executable.extend(download); + } + + let monitor_conf_upload = upload_tasks( + UploadTaskBuilderType::MonitorConf, + &Config::Cluster(config.clone()), + ); + if !monitor_conf_upload.is_empty() { + barrier.push(monitor_conf_upload.len()); + executable.extend(monitor_conf_upload); + } + + let monitor_hosts = config.get_host_as_map(); + let unpack_tasks = match component { + UpdateMonitorComponent::Grafana => monitor_hosts + .get(&DeploymentPackage::Grafana) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|host| { + UnpackFileTask::make_task_pair( + config, + &host, + config + .deployment + .monitor + .as_ref() + .unwrap() + .grafana + .as_ref() + .unwrap() + .download_url + .split('/') + .next_back() + .unwrap(), + GRAFANA_FILE_KEY, + vec![], + ) + }) + .collect::>(), + UpdateMonitorComponent::Prometheus => monitor_hosts + .get(&DeploymentPackage::Prometheus) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|host| { + UnpackFileTask::make_task_pair( + config, + &host, + config + .deployment + .monitor + .as_ref() + .unwrap() + .prometheus + .as_ref() + .unwrap() + .download_url + .split('/') + .next_back() + .unwrap(), + PROMETHEUS_FILE_KEY, + vec![], + ) + }) + .collect::>(), + UpdateMonitorComponent::NodeExporter => config + .get_unique_host_list() + .into_iter() + .map(|host| { + UnpackFileTask::make_task_pair( + config, + &host, + config + .deployment + .monitor + .as_ref() + .unwrap() + .node_exporter + .as_ref() + .unwrap() + .url + .split('/') + .next_back() + .unwrap(), + NODE_EXPORTER_FILE_KEY, + vec![], + ) + }) + .collect::>(), + }; + if !unpack_tasks.is_empty() { + barrier.push(unpack_tasks.len()); + executable.extend(unpack_tasks); + } + + let stop_cmd = SubCommand::Monitor { + cluster: match cmd { + SubCommand::Update { + cluster: Some(cluster), + .. + } => cluster.clone(), + _ => unreachable!(), + }, + command: "stop".to_string(), + }; + let start_cmd = SubCommand::Monitor { + cluster: match cmd { + SubCommand::Update { + cluster: Some(cluster), + .. + } => cluster.clone(), + _ => unreachable!(), + }, + command: "start".to_string(), + }; + + let stop_tasks = match component { + UpdateMonitorComponent::Grafana => { + MonitorCtlTask::grafana_ctl_task(stop_cmd.clone(), config) + } + UpdateMonitorComponent::Prometheus => { + MonitorCtlTask::prometheus_ctl_task(stop_cmd.clone(), config) + } + UpdateMonitorComponent::NodeExporter => { + MonitorCtlTask::exporter_ctl_task(stop_cmd.clone(), config) + } + }; + if !stop_tasks.is_empty() { + barrier.push(stop_tasks.len()); + executable.extend(stop_tasks); + } + + let start_tasks = match component { + UpdateMonitorComponent::Grafana => { + MonitorCtlTask::grafana_ctl_task(start_cmd.clone(), config) + } + UpdateMonitorComponent::Prometheus => { + MonitorCtlTask::prometheus_ctl_task(start_cmd.clone(), config) + } + UpdateMonitorComponent::NodeExporter => { + MonitorCtlTask::exporter_ctl_task(start_cmd.clone(), config) + } + }; + if !start_tasks.is_empty() { + barrier.push(start_tasks.len()); + executable.extend(start_tasks); + } + + TaskExecutionContext { + task_group: format!( + "update-monitor-{}", + Self::update_monitor_component_name(component) + ), + barrier: Some(barrier), + executable, + } + } + + async fn run_update_command( + &'static self, + cmd: SubCommand, + config: Config, + quiet: bool, + verbose: bool, + ) -> Result<()> { + let Config::Cluster(cfg) = config.clone() else { + unreachable!(); + }; + + let (download_only, monitor_component) = match &cmd { + SubCommand::Update { + download_only, + monitor, + .. + } => (*download_only, monitor.clone()), + _ => unreachable!(), + }; + + if download_only { + let tasks = DownloadTask::from_config(&cfg)?; + let context = TaskExecutionContext { + task_group: "update-download".to_string(), + barrier: Some(vec![tasks.len()]), + executable: tasks, + }; + let outfile = if quiet { + let f = fs::OpenOptions::new() + .create(true) + .truncate(true) + .open(self.home.join("task-result"))?; + Some(f) + } else { + None + }; + let task_mgr = self.task_mgr.clone(); + let recv_rs_and_print_join = tokio::task::spawn(async move { + task_mgr + .write_task_result(outfile, verbose) + .await + .expect("write task result failed"); + }); + self.task_mgr.run_context(context, config, true).await?; + recv_rs_and_print_join.await?; + println!("required tarballs downloaded into cache"); + return Ok(()); + } + + if let Some(component) = monitor_component { + let context = self.monitor_update_context(&cmd, &cfg, &component); + let outfile = if quiet { + let f = fs::OpenOptions::new() + .create(true) + .truncate(true) + .open(self.home.join("task-result"))?; + Some(f) + } else { + None + }; + let task_mgr = self.task_mgr.clone(); + let recv_rs_and_print_join = tokio::task::spawn(async move { + task_mgr + .write_task_result(outfile, verbose) + .await + .expect("write task result failed"); + }); + self.task_mgr.run_context(context, config, true).await?; + recv_rs_and_print_join.await?; + self.save_deployment_config(&cfg, true).await?; + println!( + "cluster {} monitor component {} is updated!", + cfg.deployment.cluster_name, + Self::update_monitor_component_name(&component) + ); + return Ok(()); + } + + self.run_impl_default(cmd, Some(config), quiet, verbose) + .await + } + + async fn run_impl_default( + &'static self, + mut cmd: SubCommand, + option_config: Option, + quiet: bool, + verbose: bool, + ) -> Result<()> { + let config = match option_config { + Some(config) => config, + None => self.get_config(cmd.clone()).await?, + }; + + match config { + Config::Cluster(mut deploy_config) => { + let cmd_for_match = cmd.clone(); + match cmd_for_match { + SubCommand::Connect { .. } => { + println!("{}", deploy_config.client_conn()); + } + SubCommand::Export { cluster, output } => { + let yaml_str = deploy_config.to_yaml()?; + if let Some(path) = output { + std::fs::write(path.clone(), &yaml_str) + .map_err(|e| anyhow!("failed to write {}: {}", path, e))?; + println!("Exported cluster '{}' topology to {}", cluster, path); + } else { + println!("{}", yaml_str); + } + } + _ => { + if let SubCommand::Scale { + version: requested_version, + add_nodes, + remove_nodes, + .. + } = &mut cmd + { + if let Some(version_value) = requested_version.clone() { + if add_nodes.is_empty() { + bail!("--version requires at least one entry in --add-nodes"); + } + if !remove_nodes.is_empty() { + bail!("--version cannot be combined with --remove-nodes"); + } + + let mut version_config = deploy_config.clone(); + version_config.deployment.version = Some(version_value.clone()); + version_config.deployment.tx_service.image = None; + if let Some(logsrv) = version_config.deployment.log_service.as_mut() + { + logsrv.image = None; + } + + self.resolve_version(&mut version_config.deployment).await?; + let resolved_version = + version_config.deployment.version.clone().unwrap(); + let resolved_image = + version_config.deployment.tx_service.image.clone().unwrap(); + + deploy_config.tx_version_override = Some(resolved_version.clone()); + deploy_config.tx_image_override = Some(resolved_image); + + *requested_version = Some(resolved_version); + } + } + + if let Some(noop_msg) = Self::idempotent_noop_message(&cmd, &deploy_config) + { + println!("{noop_msg}"); + return Ok(()); + } + + let task_mgr = self.task_mgr.clone(); + let outfile = if quiet { + let f = fs::OpenOptions::new() + .create(true) + .truncate(true) + .open(self.home.join("task-result"))?; + Some(f) + } else { + None + }; + + let recv_rs_and_print_join = tokio::task::spawn(async move { + task_mgr + .write_task_result(outfile, verbose) + .await + .expect("write task result failed"); + }); + + let rs = self + .task_mgr + .run_tasks(cmd.clone(), Config::Cluster(deploy_config.clone())) + .await?; + recv_rs_and_print_join.await?; + info!(r#"all tasks complete. task_size={}"#, rs.len()); + Self::print_status_output(&cmd, &deploy_config, &rs, verbose); + + let should_verify_after_finish = matches!( + &cmd, + SubCommand::Update { + cluster: Some(_), + .. + } | SubCommand::UpdateConf { .. } + | SubCommand::Scale { .. } + | SubCommand::ScaleLog { .. } + ); + let final_config = deploy_config.clone(); + let verify_cluster = match &cmd { + SubCommand::Scale { cluster, .. } + | SubCommand::ScaleLog { cluster, .. } => Some(cluster.clone()), + _ => None, + }; + self.finishing(cmd, Config::Cluster(deploy_config)).await?; + if should_verify_after_finish { + let verify_config = if let Some(cluster) = verify_cluster { + self.state_mgr + .load_deployment_from_state(&cluster) + .await? + .unwrap_or(final_config) + } else { + final_config + }; + self.ensure_critical_services_healthy( + &verify_config, + "verify mutation", + ) + .await?; + } + } + } + } + Config::Proxy(proxy_config) => { + proxy_config.connection.auth.check_keypair()?; + match cmd.clone() { + SubCommand::Proxy { .. } => { + let task_mgr = self.task_mgr.clone(); + let outfile = if quiet { + let f = fs::OpenOptions::new() + .create(true) + .truncate(true) + .open(self.home.join("task-result"))?; + Some(f) + } else { + None + }; + + let recv_rs_and_print_join = tokio::task::spawn(async move { + task_mgr + .write_task_result(outfile, verbose) + .await + .expect("write task result failed"); + }); + + let rs = self + .task_mgr + .run_tasks(cmd.clone(), Config::Proxy(proxy_config.clone())) + .await?; + recv_rs_and_print_join.await?; + info!(r#"all tasks complete. task_size={}"#, rs.len()); + + self.finishing(cmd, Config::Proxy(proxy_config)).await?; + } + _ => unreachable!(), + } + } + } + + Ok(()) + } + async fn save_deployment_config(&self, config: &DeployConfig, upsert: bool) -> Result<()> { let cluster = &config.deployment.cluster_name; let all_hosts = config.get_unique_host_list().join(";"); @@ -1008,8 +1517,27 @@ impl CmdExecutor { SubCommand::Update { cluster: Some(cluster), version, + download_only, + monitor, + monitor_url, .. } => { + if download_only && monitor.is_some() { + bail!("--download-only cannot be combined with --monitor"); + } + if download_only && version.is_none() { + bail!( + "`update --download-only` requires a target version. Use `eloqctl update {cluster} --download-only`" + ); + } + if version + .as_deref() + .is_some_and(|v| v.eq_ignore_ascii_case("download-only")) + { + bail!( + "`download-only` is a flag, not a version. Use `eloqctl update {cluster} --download-only`" + ); + } let mut config = self .state_mgr .load_deployment_from_state(&cluster) @@ -1026,6 +1554,9 @@ impl CmdExecutor { } self.resolve_version(&mut config.deployment).await?; } + if let Some(component) = monitor.as_ref() { + Self::configure_monitor_update(&mut config, component, monitor_url.clone())?; + } Self::validate_metrics_config(&config)?; Ok(Config::Cluster(config)) @@ -1156,6 +1687,20 @@ impl CmdExecutor { _ => {} } + if matches!( + &cmd, + SubCommand::Update { + cluster: Some(_), + .. + } + ) { + let config = match option_config { + Some(config) => config, + None => self.get_config(cmd.clone()).await?, + }; + return self.run_update_command(cmd, config, quiet, verbose).await; + } + // Extract cluster_config from option_config or load it let config = match option_config { Some(config) => match config { @@ -1255,16 +1800,7 @@ impl CmdExecutor { .await?; recv_rs_and_print_join.await?; info!(r#"all tasks complete. task_size={}"#, rs.len()); - if let SubCommand::Status { detail, .. } = &cmd { - if !detail { - Self::print_status_summary(&rs); - if !verbose { - println!( - "Tip: use `--verbose` to show per-task execution details." - ); - } - } - } + Self::print_status_output(&cmd, &deploy_config, &rs, verbose); // Using cluster_config again without moving it let should_verify_after_finish = matches!( @@ -1590,95 +2126,41 @@ impl CmdExecutor { product: Option, store: Option, ) -> Result<()> { - let client = self.pg_client().await?; - let arch = cpu_arch(); - let os = self.os_vers(); - let product_name = product.as_ref().map(|p| p.name().to_string()); let store_name = store.as_ref().map(|s| s.to_string()); - - let mut sql = "SELECT * FROM tx_release WHERE arch=$1 AND os=$2".to_owned(); - let mut params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = vec![&arch, &os]; - if let Some(ref p) = product_name { - sql.push_str(&format!(" AND product=${}", params.len() + 1)); - params.push(p); - } - if let Some(ref s) = store_name { - sql.push_str(&format!(" AND store=${}", params.len() + 1)); - params.push(s); - } - - let list = client - .query(&sql, ¶ms) - .await? - .into_iter() - .map(|row| { - let product: String = row.get("product"); - let store: String = row.get("store"); - let major: i32 = row.get("version_major"); - let minor: i32 = row.get("version_minor"); - let build: i32 = row.get("version_build"); - let version: String = format!("{major}.{minor}.{build}"); - VersionRow { - product, - store, - version, - } - }) - .collect_vec(); + let releases = fetch_eloqkv_releases(&HTTP_CLIENT).await?; + let list = list_versions_from_releases(&releases, product, store_name.as_deref()); let table = tabled::Table::new(list); println!("{table}\n"); Ok(()) } pub async fn resolve_version(&self, cnf: &mut Deployment) -> Result<()> { - let product = cnf.product().name().to_owned(); - let arch = cpu_arch(); let os = self.os_vers(); + let arch = cpu_arch(); // Get store name once and reuse it, if not set, use rocksdb as default let store = cnf .storage_service .as_ref() .map_or("rocksdb".to_string(), |s| s.pretty_name()); + let releases = fetch_eloqkv_releases(&HTTP_CLIENT).await?; if cnf.version.is_some() && cnf.version_str().eq_ignore_ascii_case("latest") { - let client = self.pg_client().await?; - - // Use the correct query based on storage_service existence - let row = if cnf.storage_service.is_some() { - client.query_one( - "SELECT * FROM tx_release WHERE product=$1 AND arch=$2 AND os=$3 AND store=$4 - ORDER BY version_major DESC,version_minor DESC,version_build DESC LIMIT 1", - &[&product, &arch, &os, &store] - ).await - } else { - client - .query_one( - "SELECT * FROM tx_release WHERE product=$1 AND arch=$2 AND os=$3 - ORDER BY version_major DESC,version_minor DESC,version_build DESC LIMIT 1", - &[&product, &arch, &os], - ) - .await - } - .map_err(|e| anyhow!("fetch latest version failed: {e}"))?; - - if row.is_empty() { - bail!("no available release found") - } - let major: i32 = row.get("version_major"); - let minor: i32 = row.get("version_minor"); - let build: i32 = row.get("version_build"); - let latest: String = format!("{major}.{minor}.{build}"); + let latest = + list_versions_from_releases(&releases, Some(cnf.product()), Some(store.as_str())) + .into_iter() + .map(|row: VersionRow| row.version) + .max_by(|a, b| version_digits(a).ok().cmp(&version_digits(b).ok())) + .ok_or_else(|| { + anyhow!("no available GitHub release found for store={store}") + })?; info!("latest release version = {latest}"); cnf.version = Some(latest); } - let mut prefix = PathBuf::from(CDN); - prefix.push(&product); - let prefix = prefix.as_path().to_str().unwrap(); if cnf.tx_service.image.is_none() { let vers = cnf.version.as_deref().expect("version is missing"); - let img = format!("{prefix}/{store}/{product}-{vers}-{os}-{arch}.tar.gz"); + let img = find_eloqkv_asset(&releases, cnf.product(), vers, &store, &os, &arch)?.url; info!("tx service image is set: {img}"); cnf.tx_service.image = Some(img); } @@ -1686,7 +2168,7 @@ impl CmdExecutor { if logsrv.image.is_none() { let vers = cnf.version.as_deref().expect("version is missing"); let img = - format!("{prefix}/logservice/{store}/log-service-{vers}-{os}-{arch}.tar.gz"); + find_product_asset(&releases, "log-service", vers, &store, &os, &arch)?.url; info!("log service image is set: {img}"); logsrv.image = Some(img); } diff --git a/src/cluster_mgr/src/cli/mod.rs b/src/cluster_mgr/src/cli/mod.rs index cb89d0f7..75406c5a 100644 --- a/src/cluster_mgr/src/cli/mod.rs +++ b/src/cluster_mgr/src/cli/mod.rs @@ -36,6 +36,13 @@ pub struct Command { pub subcmd: Option, } +#[derive(Clone, Debug, Hash, PartialEq, Eq, clap::ValueEnum)] +pub enum UpdateMonitorComponent { + Grafana, + Prometheus, + NodeExporter, +} + #[derive(Subcommand, Clone, Debug, Hash, PartialEq, Eq, AsRefStr)] #[command(next_line_help = true)] pub enum SubCommand { @@ -120,17 +127,47 @@ pub enum SubCommand { detail: bool, }, - #[command(long_about = "Update cluster version. This will stop/update/start the cluster")] + #[command(long_about = "Update an existing cluster to a target version.\n\ + By default this performs a rolling upgrade that stops, replaces, and restarts services.\n\ + Use `--download-only` to resolve the requested version and download the required EloqKV tarballs\n\ + into the local cache without changing remote hosts or cluster state.\n\ + Use `--monitor` to update only one monitor component without touching the EloqKV cluster.")] #[strum(serialize = "update")] Update { + #[arg( + help = "Existing cluster name; required for rolling updates, download-only, and monitor-only updates" + )] cluster: Option, + #[arg( + help = "Target EloqKV version or `latest`; required unless using --monitor without a version change" + )] version: Option, - #[arg(long, value_name = "password for graceful shutdown")] + #[arg( + long, + default_value_t = false, + help = "Download the target release tarballs into the local cache only; do not update remote hosts. Use as: `eloqctl update --download-only`" + )] + download_only: bool, + #[arg( + long, + value_enum, + help = "Update only the selected monitor component instead of the EloqKV cluster" + )] + monitor: Option, + #[arg( + long, + help = "Override the tarball URL for the selected monitor component; valid only with --monitor" + )] + monitor_url: Option, + #[arg( + long, + value_name = "password for graceful shutdown and redis operations" + )] password: Option, #[arg( short, long, - value_name = "enable this when cluster cannot perform graceful shutdown or is already down", + value_name = "enable this when graceful shutdown is impossible or the cluster is already down", default_value_t = false )] force: bool, diff --git a/src/cluster_mgr/src/cli/task/download_task.rs b/src/cluster_mgr/src/cli/task/download_task.rs index 6fdf0689..d07ddf3d 100644 --- a/src/cluster_mgr/src/cli/task/download_task.rs +++ b/src/cluster_mgr/src/cli/task/download_task.rs @@ -14,9 +14,13 @@ use futures::stream::StreamExt; use indexmap::IndexMap; use indicatif::{MultiProgress, ProgressBar}; use itertools::Itertools; +use reqwest::header::{ACCEPT, USER_AGENT}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::ffi::{OsStr, OsString}; use std::fs; +use std::io::Read; use std::io::Write; use std::path::PathBuf; use tracing::{error, info}; @@ -137,6 +141,74 @@ impl DownloadTask { }) .collect::>() } + + async fn expected_digest(&self) -> Result> { + let download_url = DownloadUrl::from_url_str(&self.url)?; + let DownloadUrl::Remote(url) = download_url else { + return Ok(None); + }; + if url.domain() != Some("github.com") { + return Ok(None); + } + + let segments = url + .path_segments() + .map(|segments| segments.collect_vec()) + .unwrap_or_default(); + if segments.len() < 8 + || segments[0] != "eloqdata" + || segments[1] != "eloqkv" + || segments[2] != "releases" + || segments[3] != "download" + { + return Ok(None); + } + + let tag = segments[4]; + let file_name = segments.last().unwrap(); + let api_url = format!("https://api.github.com/repos/eloqdata/eloqkv/releases/tags/{tag}"); + let response = HTTP_CLIENT + .get(api_url) + .header(USER_AGENT, "eloqctl") + .header(ACCEPT, "application/vnd.github+json") + .send() + .await?; + if !response.status().is_success() { + return Ok(None); + } + + #[derive(Debug, Deserialize)] + struct ReleaseResponse { + assets: Vec, + } + + #[derive(Debug, Deserialize)] + struct GitHubAssetEntry { + name: String, + digest: Option, + } + + let release = response.json::().await?; + Ok(release + .assets + .into_iter() + .find(|asset| asset.name == *file_name) + .and_then(|asset| asset.digest)) + } + + fn sha256_file(path: &PathBuf) -> Result { + let mut file = std::fs::File::open(path)?; + let mut hasher = Sha256::new(); + let mut buf = [0u8; 8192]; + loop { + let read = file.read(&mut buf)?; + if read == 0 { + break; + } + hasher.update(&buf[..read]); + } + Ok(format!("sha256:{:x}", hasher.finalize())) + } } #[async_trait::async_trait] @@ -163,6 +235,7 @@ impl TaskExecutor for DownloadTask { return Err(anyhow!(DownloadErr(self.url.clone(), err.to_string()))); } let save_path = save_dir.join(&self.name); + let expected_digest = self.expected_digest().await.ok().flatten(); // Try HEAD first to check remote content-length for cache validation let remote_len = HTTP_CLIENT @@ -172,16 +245,33 @@ impl TaskExecutor for DownloadTask { .ok() .and_then(|r| r.content_length()); - // Use cache if file exists and size matches remote - if let Some(expected_len) = remote_len { - if save_path.exists() - && fs::metadata(&save_path) - .map(|m| m.len() == expected_len) - .unwrap_or(false) + if save_path.exists() { + let digest_matches = match &expected_digest { + Some(expected) => Self::sha256_file(&save_path) + .map(|actual| actual == *expected) + .unwrap_or(false), + None => false, + }; + + if digest_matches { + info!( + "local file cache {:?} found (sha256 matches), skipping download.", + save_path + ); + return Ok(None); + } + + if expected_digest.is_none() + && remote_len.is_some_and(|expected_len| { + fs::metadata(&save_path) + .map(|m| m.len() == expected_len) + .unwrap_or(false) + }) { info!( "local file cache {:?} found ({} bytes, matches remote), skipping download.", - save_path, expected_len + save_path, + remote_len.unwrap() ); return Ok(None); } @@ -200,13 +290,22 @@ impl TaskExecutor for DownloadTask { let file_len = response.content_length().unwrap(); // Double-check cache after successful GET (in case of concurrent download) - if save_path.exists() - && fs::metadata(&save_path) - .map(|m| m.len() == file_len) - .unwrap_or(false) - { - info!("local file cache {:?} found.", save_path); - return Ok(None); + if save_path.exists() { + let digest_matches = match &expected_digest { + Some(expected) => Self::sha256_file(&save_path) + .map(|actual| actual == *expected) + .unwrap_or(false), + None => false, + }; + if digest_matches + || (expected_digest.is_none() + && fs::metadata(&save_path) + .map(|m| m.len() == file_len) + .unwrap_or(false)) + { + info!("local file cache {:?} found.", save_path); + return Ok(None); + } } // TODO(zhanghao): Use HTTP range header to resume download let part_path = append_ext(save_path.clone(), "partial"); @@ -232,6 +331,19 @@ impl TaskExecutor for DownloadTask { } self.pg_bar .finish_with_message(format!("{} Downloaded!", self.name)); + + if let Some(expected) = &expected_digest { + let actual = Self::sha256_file(&part_path) + .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + if &actual != expected { + let _ = fs::remove_file(&part_path); + return Err(anyhow!(DownloadErr( + self.url.clone(), + format!("sha256 mismatch: expected {expected}, got {actual}") + ))); + } + } + fs::rename(part_path, save_path.as_path()) .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; diff --git a/src/cluster_mgr/src/cli/task/group/deployment_group.rs b/src/cluster_mgr/src/cli/task/group/deployment_group.rs index 5ab8d263..894a7c1e 100644 --- a/src/cluster_mgr/src/cli/task/group/deployment_group.rs +++ b/src/cluster_mgr/src/cli/task/group/deployment_group.rs @@ -2,8 +2,8 @@ use crate::cli::task::download_task::DownloadTask; use crate::cli::task::exec_custom_cmd::ExecCustomCommand; use crate::cli::task::group::{Config, DeploymentTaskGroup, TaskGroup}; use crate::cli::task::local_copy_task::LocalCopyTask; +use crate::cli::task::local_extract_task::LocalExtractTask; use crate::cli::task::task_base::{TaskExecutionContext, TaskId, TaskInstance}; -use crate::cli::task::unpack_file_task::UnpackFileTask; use crate::cli::task::upload::upload_task_builder::{upload_tasks, UploadTaskBuilderType}; use crate::cli::SubCommand; use crate::config::config_base::DEPLOYMENT_CHECK_SUCCESS_TASK; @@ -71,22 +71,14 @@ impl TaskGroup for DeploymentTaskGroup { } else { true }; - let (db_upload_task, unpack_task) = if need_skip_success_task { - ( - DeploymentTaskGroup::skip_success_task_execution( - &upload_tasks(UploadTaskBuilderType::EloqAll, config), - &success_task_vec, - ), - DeploymentTaskGroup::skip_success_task_execution( - &UnpackFileTask::from_config(cluster_config)?, - &success_task_vec, - ), + let local_extract_task = LocalExtractTask::from_config(cluster_config)?; + let db_upload_task = if need_skip_success_task { + DeploymentTaskGroup::skip_success_task_execution( + &upload_tasks(UploadTaskBuilderType::EloqAll, config), + &success_task_vec, ) } else { - ( - upload_tasks(UploadTaskBuilderType::EloqAll, config), - UnpackFileTask::from_config(cluster_config)?, - ) + upload_tasks(UploadTaskBuilderType::EloqAll, config) }; let upload_tx_conf = upload_tasks(UploadTaskBuilderType::TxConf, config); @@ -109,16 +101,16 @@ impl TaskGroup for DeploymentTaskGroup { let barrier = Some(vec![ mkdir_remote_dir.len(), copy_or_download_task_instances.len(), + local_extract_task.len(), db_upload_task.len(), - unpack_task.len(), upload_tx_conf.len(), upload_monitor_conf_tasks.len(), ]); let mut executable = IndexMap::new(); executable.extend(mkdir_remote_dir); executable.extend(copy_or_download_task_instances); + executable.extend(local_extract_task); executable.extend(db_upload_task); - executable.extend(unpack_task); executable.extend(upload_tx_conf); executable.extend(upload_monitor_conf_tasks); Ok(TaskExecutionContext { diff --git a/src/cluster_mgr/src/cli/task/group/scale_group.rs b/src/cluster_mgr/src/cli/task/group/scale_group.rs index 2f427981..36cd7d40 100644 --- a/src/cluster_mgr/src/cli/task/group/scale_group.rs +++ b/src/cluster_mgr/src/cli/task/group/scale_group.rs @@ -5,6 +5,7 @@ use crate::cli::task::eloq_tx_ctl_task::{EloqTxCtlTask, ServerType}; use crate::cli::task::exec_custom_cmd::ExecCustomCommand; use crate::cli::task::group::{Config, ScaleTaskGroup}; use crate::cli::task::install_dep_pkg::DepPkgTask; +use crate::cli::task::local_extract_task::LocalExtractTask; use crate::cli::task::redis_op_task::{ClusterNodes, RedisOpTask}; use crate::cli::task::scale_op_task::{ScaleOpConfig, ScaleOpTask}; use crate::cli::task::ssh_check_task::SshCheckTask; @@ -13,14 +14,13 @@ use crate::cli::task::task_utils::{ClusterNodesWithConfig, ScaleOperationType}; use crate::cli::task::topology_display_task::TopologyDisplayTask; use crate::cli::task::topology_update_task::TopologyUpdateTask; use crate::cli::task::tx_conf_update_task::TxConfUpdateTask; -use crate::cli::task::unpack_file_task::UnpackFileTask; use crate::cli::task::upload::upload_task_builder::{ build_task_instance, get_source_host, upload_tasks, upload_tasks_with_nodes, UploadTaskBuilderType, }; -use crate::cli::{download_dir, SubCommand}; -use crate::config::config_base::UploadFile; -use crate::config::DeploymentPackage; +use crate::cli::SubCommand; +use crate::config::config_base::{UploadFile, ELOQ_FILE_KEY}; +use crate::config::{DeploymentPackage, DownloadUrl}; use anyhow::{anyhow, bail}; use async_trait::async_trait; use indexmap::IndexMap; @@ -177,11 +177,6 @@ impl super::TaskGroup for ScaleTaskGroup { .as_ref() .cloned() .unwrap_or_else(|| deploy_config.deployment.tx_image().to_string()); - let tx_image_filename = effective_tx_image_url - .split('/') - .next_back() - .unwrap_or("") - .to_string(); let effective_version = deploy_config .tx_version_override .as_ref() @@ -245,16 +240,6 @@ impl super::TaskGroup for ScaleTaskGroup { (ScaleOperationType::RemoveNodes, remove_nodes, None) }; - if version.is_some() { - let download_tasks = DownloadTask::instances(DownloadTask::from_urls(vec![ - effective_tx_image_url.clone(), - ])); - if !download_tasks.is_empty() { - barrier.push(download_tasks.len()); - executable.extend(download_tasks); - } - } - let candidate_nodes_after_scale = match operation_type { ScaleOperationType::AddNodes => { let mut after = candidate_nodes_before_scale.clone(); @@ -398,74 +383,62 @@ impl super::TaskGroup for ScaleTaskGroup { } if !new_nodes_to_upload_tarball.is_empty() { - if tx_image_filename.is_empty() { + let tx_download_url = DownloadUrl::from_url_str(&effective_tx_image_url)?; + let tx_home = deploy_config.deployment.product().home().to_string(); + + let download_tasks = DownloadTask::instances(DownloadTask::from_urls(vec![ + effective_tx_image_url.clone(), + ])); + if !download_tasks.is_empty() { + barrier.push(download_tasks.len()); + executable.extend(download_tasks); + } + + let extract_tasks = LocalExtractTask::from_urls(vec![( + ELOQ_FILE_KEY.to_string(), + tx_download_url.clone(), + tx_home.clone(), + )]); + if !extract_tasks.is_empty() { + barrier.push(extract_tasks.len()); + executable.extend(extract_tasks); + } + + let staged_dir = LocalExtractTask::staged_dir_for(&tx_download_url, &tx_home); + if !staged_dir.exists() { bail!( - "Unable to determine TX service image filename from {}", - effective_tx_image_url + "TX service staged directory not found: {}", + staged_dir.display() ); } info!( - "Preparing to upload TX service tarball '{}' to {} new hosts", - tx_image_filename, + "Preparing to sync TX service directory '{}' to {} new hosts", + staged_dir.display(), new_nodes_to_upload_tarball.len() ); - let download_dir = download_dir(); - let product_dir = "eloqkv"; - let store = deploy_config - .deployment - .storage_service - .as_ref() - .map_or("rocksdb".to_string(), |s| s.pretty_name()); - let tarball_path = download_dir - .join(product_dir) - .join(&store) - .join(&tx_image_filename); - - if tarball_path.exists() { - for host in &new_hosts_to_upload_tarball { - let source_host = get_source_host(None); - let upload_file = UploadFile { - source: tarball_path.to_string_lossy().to_string(), - dest: deploy_config.install_dir(), - extension: "gz".to_string(), - host: host.clone(), - copy_dir: false, - }; - - let (id, instance) = build_task_instance( - source_host, - upload_file, - config, - "deploy", - "tx_service_upload", - ); - barrier.push(1); - executable.insert(id, instance); - info!("Added upload task for TX service tarball to host {}", host); - } - - let mut temp_config = deploy_config.clone(); - temp_config.deployment.tx_service.tx_host_ports = - new_nodes_to_upload_tarball.clone(); - temp_config.deployment.tx_service.standby_host_ports = None; - temp_config.deployment.tx_service.voter_host_ports = None; - temp_config.deployment.log_service = None; - temp_config.deployment.tx_service.image = Some(effective_tx_image_url.clone()); - temp_config.tx_image_override = Some(effective_tx_image_url.clone()); - - let tx_unpack_tasks = UnpackFileTask::unpack_eloqservers(&temp_config); - for (task_id, instance) in tx_unpack_tasks { - info!("Added unpack task for TX service on host {}", task_id.host); - barrier.push(1); - executable.insert(task_id, instance); - } - } else { - bail!( - "TX service tarball not found at expected path: {}", - tarball_path.display() + for host in &new_hosts_to_upload_tarball { + let source_host = get_source_host(None); + let upload_file = UploadFile { + source: staged_dir.to_string_lossy().to_string(), + dest: format!("{}/{}", deploy_config.install_dir(), tx_home), + extension: "dir".to_string(), + host: host.clone(), + copy_dir: true, + delete_remote: true, + }; + + let (id, instance) = build_task_instance( + source_host, + upload_file, + config, + "deploy", + "tx_service_upload", ); + barrier.push(1); + executable.insert(id, instance); + info!("Added TX service sync task for host {}", host); } } } diff --git a/src/cluster_mgr/src/cli/task/group/scale_log_group.rs b/src/cluster_mgr/src/cli/task/group/scale_log_group.rs index 285184af..a606defd 100644 --- a/src/cluster_mgr/src/cli/task/group/scale_log_group.rs +++ b/src/cluster_mgr/src/cli/task/group/scale_log_group.rs @@ -1,8 +1,10 @@ use crate::cli::task::db_update_log_task::DbDeploymentUpdateLogTask; +use crate::cli::task::download_task::DownloadTask; use crate::cli::task::eloq_log_ctl_task::{EloqLogCtlTask, LogCtlCmd}; use crate::cli::task::eloq_log_probe_task::EloqLogProbeTask; use crate::cli::task::exec_custom_cmd::ExecCustomCommand; use crate::cli::task::group::{Config, ScaleLogTaskGroup, TaskGroup}; +use crate::cli::task::local_extract_task::LocalExtractTask; use crate::cli::task::redis_op_task::{ClusterNodes, RedisOpTask}; use crate::cli::task::scale_log_cleanup_task::ScaleLogCleanupTask; use crate::cli::task::scale_log_op_task::ScaleLogOpTask; @@ -15,7 +17,7 @@ use crate::cli::task::topology_display_task::TopologyDisplayTask; use crate::cli::task::topology_update_task::TopologyUpdateTask; use crate::cli::task::upload::upload_task_builder::{build_task_instance, get_source_host}; use crate::cli::SubCommand; -use crate::config::config_base::{UploadFile, LOG_SERVICE_HOME}; +use crate::config::config_base::{UploadFile, ELOQ_LOG_FILE_KEY, LOG_SERVICE_HOME}; use crate::config::log_service::{LogProcessKey, LogServiceNode}; use crate::config::DownloadUrl; use anyhow::anyhow; @@ -23,10 +25,8 @@ use async_trait::async_trait; use indexmap::IndexMap; use itertools::Itertools; use std::collections::HashMap; -use std::env; -use std::path::PathBuf; use tokio::sync::watch; -use tracing::{info, warn}; +use tracing::info; use uuid::Uuid; #[async_trait] @@ -216,18 +216,6 @@ impl TaskGroup for ScaleLogTaskGroup { let log_scripts_paths = temp_config.gen_log_start_script()?; info!("Generated log scripts: {:?}", log_scripts_paths); - // Extract existing host names from the original config - needed to identify truly new hosts - let existing_hosts: Vec = deploy_cfg - .deployment - .log_service - .as_ref() - .unwrap() - .nodes - .iter() - .map(|node| node.host.clone()) - .unique() - .collect(); - // Extract all host names with ports from the temp_log_service (includes both existing and new nodes) let all_log_nodes: Vec = temp_log_service .nodes @@ -263,46 +251,58 @@ impl TaskGroup for ScaleLogTaskGroup { barrier.push(mkdir_remote_dir.len()); executable.extend(mkdir_remote_dir); - // Step 5: Create upload tasks for the newly added log nodes scripts - info!("Setting up upload tasks for log start scripts"); + if !hosts_to_upload_log_binary.is_empty() { + let log_image = deploy_cfg + .deployment + .log_image() + .ok_or_else(|| anyhow!("Log service image not configured"))?; + let log_download_url = DownloadUrl::from_url_str(log_image)?; + + let download_tasks = + DownloadTask::instances(DownloadTask::from_urls(vec![ + log_image.to_string() + ])); + if !download_tasks.is_empty() { + barrier.push(download_tasks.len()); + executable.extend(download_tasks); + } - // Upload log start bash file for all nodes - for node_str in &node_list_to_upload_bash { - if let Some((host, _port_str)) = node_str.split_once(':') { - // Gather all start_tx_log_*.bash scripts for this host - let mut sources = Vec::new(); - if let Some(script_list) = &log_scripts_paths { - for path in script_list { - if path.to_string_lossy().contains(host) { - sources.push(path.to_string_lossy().to_string()); - info!("Adding script to upload: {}", path.display()); - } - } - } + let extract_tasks = LocalExtractTask::from_urls(vec![( + ELOQ_LOG_FILE_KEY.to_string(), + log_download_url.clone(), + LOG_SERVICE_HOME.to_string(), + )]); + if !extract_tasks.is_empty() { + barrier.push(extract_tasks.len()); + executable.extend(extract_tasks); + } - info!( - "Found {} files to upload binary for host:port {}", - sources.len(), - node_str - ); + let staged_dir = + LocalExtractTask::staged_dir_for(&log_download_url, LOG_SERVICE_HOME); + if !staged_dir.exists() { + return Err(anyhow!( + "Log service staged directory not found: {}", + staged_dir.display() + )); + } - // Create an upload file that includes all sources + for host in &hosts_to_upload_log_binary { let upload_file = UploadFile { - source: sources.join(" "), - dest: deploy_cfg.install_dir(), - extension: "bash".to_string(), + source: staged_dir.to_string_lossy().to_string(), + dest: format!("{}/{}", deploy_cfg.install_dir(), LOG_SERVICE_HOME), + extension: "dir".to_string(), host: host.to_string(), - copy_dir: false, + copy_dir: true, + delete_remote: true, }; - // Create the upload task let source_host = get_source_host(None); let (id, instance) = build_task_instance( source_host, upload_file, config, "deploy", - &format!("deploy_log_start_bash_to_{}", node_str), + "deploy_eloq_log_dir", ); barrier.push(1); @@ -310,99 +310,50 @@ impl TaskGroup for ScaleLogTaskGroup { } } - // Upload log binary tarball for newly added hosts - for host in &hosts_to_upload_log_binary { - let mut sources = Vec::new(); - - // Add the log service tarball if it exists in the deployment configuration - if let Some(log_image) = deploy_cfg.deployment.log_image() { - if let Ok(download_url) = DownloadUrl::from_url_str(log_image) { - let file_name = download_url.file_name(); - // Check if the file exists in the download directory - let download_dir = PathBuf::from(format!( - "{}/.eloqctl/download", - env::var("HOME").unwrap_or_else(|_| "/home/eloq".to_string()) - )); - let file_path = download_dir - .join("eloqkv") - .join("logservice") - .join(&file_name); - - if file_path.exists() { - info!("Found log service tarball at {}", file_path.display()); - sources.push(file_path.to_string_lossy().to_string()); - } else { - warn!( - "Log service tarball not found at expected path: {}", - file_path.display() - ); - } - } - } - - info!( - "Found {} files to upload binary for host {}", - sources.len(), - host - ); - - // Create an upload file that includes all sources - let upload_file = UploadFile { - source: sources.join(" "), - dest: deploy_cfg.install_dir(), - extension: "gz".to_string(), - host: host.to_string(), - copy_dir: false, - }; - - // Create the upload task - - let source_host = get_source_host(None); - let (id, instance) = build_task_instance( - source_host, - upload_file, - config, - "deploy", - "deploy_eloq_all_gz", - ); - - barrier.push(1); - executable.insert(id, instance); - } - - // Step 6: Create unpack tasks for the log service tarball on new nodes + // Step 5: Create upload tasks for the newly added log nodes scripts + info!("Setting up upload tasks for log start scripts"); - // Filter for only truly new hosts (in hosts_to_upload but not in existing_hosts) - let new_log_hosts = hosts_to_upload_log_binary + // Upload log start bash file for all nodes + for host in node_list_to_upload_bash .iter() - .filter(|h| !existing_hosts.contains(&h.to_string())) - .map(|h| h.to_string()) - .collect::>(); - - if !new_log_hosts.is_empty() { - // Create a temporary deployment config with just these new hosts as log nodes - let mut temp_config = temp_config.clone(); - if let Some(log_srv) = &mut temp_config.deployment.log_service { - // Filter the nodes to only include the new hosts - log_srv - .nodes - .retain(|node| new_log_hosts.contains(&node.host)); - } + .filter_map(|node| node.split_once(':').map(|(host, _)| host.to_string())) + .unique() + { + if let Some(script_list) = &log_scripts_paths { + for path in script_list { + if !path.to_string_lossy().contains(&host) { + continue; + } + info!("Adding script to upload: {}", path.display()); + let upload_file = UploadFile { + source: path.to_string_lossy().to_string(), + dest: deploy_cfg.install_dir(), + extension: "bash".to_string(), + host: host.clone(), + copy_dir: false, + delete_remote: false, + }; - // Generate the unpack tasks - only unpack logservice, not eloqkv - let log_unpack_tasks = - crate::cli::task::unpack_file_task::UnpackFileTask::unpack_log_servers( - &temp_config, - ); + let task_name = path + .file_name() + .and_then(|name| name.to_str()) + .map(|name| format!("deploy_{}", name.replace('.', "_"))) + .unwrap_or_else(|| { + format!("deploy_log_start_bash_to_{}", host) + }); + let source_host = get_source_host(None); + let (id, instance) = build_task_instance( + source_host, + upload_file, + config, + "deploy", + &task_name, + ); - // Add the tasks to the executable - for (task_id, instance) in log_unpack_tasks { - info!("Added unpack task for log service on host {}", task_id.host); - barrier.push(1); - executable.insert(task_id, instance); + barrier.push(1); + executable.insert(id, instance); + } } - } else { - info!("No new log hosts to unpack"); } // Step 7: Use EloqLogCtlTask to start the new log nodes @@ -811,46 +762,45 @@ impl TaskGroup for ScaleLogTaskGroup { info!("Setting up upload tasks for updated log start scripts"); // Upload log start bash file for all remaining nodes - for node_str in &node_list_to_upload_bash { - if let Some((host, _port_str)) = node_str.split_once(':') { - // Gather all start_tx_log_*.bash scripts for this host - let mut sources = Vec::new(); - if let Some(script_list) = &log_scripts_paths { - for path in script_list { - if path.to_string_lossy().contains(host) { - sources.push(path.to_string_lossy().to_string()); - info!("Adding script to upload: {}", path.display()); - } + for host in node_list_to_upload_bash + .iter() + .filter_map(|node| node.split_once(':').map(|(host, _)| host.to_string())) + .unique() + { + if let Some(script_list) = &log_scripts_paths { + for path in script_list { + if !path.to_string_lossy().contains(&host) { + continue; } - } - - info!( - "Found {} files to upload for host:port {}", - sources.len(), - node_str - ); - - // Create an upload file that includes all sources - let upload_file = UploadFile { - source: sources.join(" "), - dest: deploy_cfg.install_dir(), - extension: "bash".to_string(), - host: host.to_string(), - copy_dir: false, - }; + info!("Adding script to upload: {}", path.display()); + let upload_file = UploadFile { + source: path.to_string_lossy().to_string(), + dest: deploy_cfg.install_dir(), + extension: "bash".to_string(), + host: host.clone(), + copy_dir: false, + delete_remote: false, + }; - // Create the upload task - let source_host = get_source_host(None); - let (id, instance) = build_task_instance( - source_host, - upload_file, - config, - "deploy", - &format!("deploy_log_start_bash_to_{}", node_str), - ); + let task_name = path + .file_name() + .and_then(|name| name.to_str()) + .map(|name| format!("deploy_{}", name.replace('.', "_"))) + .unwrap_or_else(|| { + format!("deploy_log_start_bash_to_{}", host) + }); + let source_host = get_source_host(None); + let (id, instance) = build_task_instance( + source_host, + upload_file, + config, + "deploy", + &task_name, + ); - barrier.push(1); - executable.insert(id, instance); + barrier.push(1); + executable.insert(id, instance); + } } } } diff --git a/src/cluster_mgr/src/cli/task/local_extract_task.rs b/src/cluster_mgr/src/cli/task/local_extract_task.rs new file mode 100644 index 00000000..6d15467c --- /dev/null +++ b/src/cluster_mgr/src/cli/task/local_extract_task.rs @@ -0,0 +1,189 @@ +use crate::cli::task::task_base::{ + CmdErr, ExecutionValue, TaskArgValue, TaskExecutor, TaskHost, TaskId, TaskInstance, +}; +use crate::cli::{download_dir, CMD, CMD_OUTPUT, CMD_STATUS}; +use crate::config::config_base::{ + DeployConfig, ELOQ_FILE_KEY, ELOQ_LOG_FILE_KEY, GRAFANA_FILE_KEY, LOG_SERVICE_HOME, + NODE_EXPORTER_FILE_KEY, PROMETHEUS_FILE_KEY, +}; +use crate::config::DownloadUrl; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use flate2::read::GzDecoder; +use indexmap::IndexMap; +use itertools::Itertools; +use std::collections::HashMap; +use std::fs; +use std::fs::File; +use std::path::{Path, PathBuf}; +use tar::Archive; +use tracing::info; + +const ARCHIVE_PATH: &str = "_archive_path"; +const EXTRACT_ROOT: &str = "_extract_root"; +const UNPACK_DEST: &str = "_unpack_dest"; + +#[derive(Clone, Debug)] +pub struct LocalExtractTask { + task_id: TaskId, +} + +impl LocalExtractTask { + pub fn from_config(config: &DeployConfig) -> Result> { + let links = config.deployment.all_download_links()?; + let entries = links + .iter() + .filter_map(|(key, url)| { + Self::unpack_dest_for_key(config, key).map(|dest| (key.clone(), url.clone(), dest)) + }) + .collect_vec(); + Ok(Self::instances(entries)) + } + + pub fn from_urls( + entries: Vec<(String, DownloadUrl, String)>, + ) -> IndexMap { + Self::instances(entries) + } + + fn instances(entries: Vec<(String, DownloadUrl, String)>) -> IndexMap { + entries + .into_iter() + .unique_by(|(_key, url, dest)| (url.file_name(), dest.clone())) + .map(|(_key, url, unpack_dest)| { + let file_name = url.file_name(); + let task_id = TaskId { + cmd: "extract".to_string(), + task: format!("{}_extract", file_name), + host: "_local".to_string(), + }; + let archive_path = Self::cached_archive_path(&url); + let extract_root = Self::extract_root_for(&url); + ( + task_id.clone(), + TaskInstance { + task_input: HashMap::from([ + ( + ARCHIVE_PATH.to_string(), + TaskArgValue::Str(archive_path.to_string_lossy().to_string()), + ), + ( + EXTRACT_ROOT.to_string(), + TaskArgValue::Str(extract_root.to_string_lossy().to_string()), + ), + (UNPACK_DEST.to_string(), TaskArgValue::Str(unpack_dest)), + ]), + task: Box::new(Self::new(task_id)), + task_host: TaskHost::Local, + }, + ) + }) + .collect() + } + + pub fn new(task_id: TaskId) -> Self { + Self { task_id } + } + + pub fn cached_archive_path(url: &DownloadUrl) -> PathBuf { + match url { + DownloadUrl::Local(_) => download_dir().join(url.file_name()), + DownloadUrl::Remote(_) => PathBuf::from(url.cache_dir().unwrap()).join(url.file_name()), + } + } + + pub fn extract_root_for(url: &DownloadUrl) -> PathBuf { + let mut path = match url { + DownloadUrl::Local(_) => download_dir(), + DownloadUrl::Remote(_) => PathBuf::from(url.cache_dir().unwrap()), + }; + path.push("_extracted"); + path.push(url.file_name().trim_end_matches(".tar.gz")); + path + } + + pub fn staged_dir_for(url: &DownloadUrl, unpack_dest: &str) -> PathBuf { + Self::extract_root_for(url).join(unpack_dest) + } + + pub fn unpack_dest_for_key(config: &DeployConfig, key: &str) -> Option { + match key { + ELOQ_FILE_KEY => Some(config.deployment.product().home().to_string()), + ELOQ_LOG_FILE_KEY => Some(LOG_SERVICE_HOME.to_string()), + PROMETHEUS_FILE_KEY => Some("prometheus".to_string()), + GRAFANA_FILE_KEY => Some("grafana".to_string()), + NODE_EXPORTER_FILE_KEY => Some("node_exporter".to_string()), + _ => None, + } + } + + fn unpack_archive(archive_path: &Path, destination: &Path) -> Result<()> { + let file = File::open(archive_path)?; + let decoder = GzDecoder::new(file); + let mut archive = Archive::new(decoder); + + for entry in archive.entries()? { + let mut entry = entry?; + let entry_path = entry.path()?; + let stripped = entry_path.components().skip(1).collect::(); + if stripped.as_os_str().is_empty() { + continue; + } + let out_path = destination.join(stripped); + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent)?; + } + entry.unpack(&out_path)?; + } + Ok(()) + } +} + +#[async_trait] +impl TaskExecutor for LocalExtractTask { + fn identifier(&self) -> TaskId { + self.task_id.clone() + } + + async fn execute( + &self, + _task_host: TaskHost, + task_arg: HashMap, + ) -> Result> { + info!("execute {}", self.task_id.format_string()); + let archive_path = PathBuf::from(TaskArgValue::into_inner_value::( + task_arg.get(ARCHIVE_PATH).unwrap().clone(), + )); + let extract_root = PathBuf::from(TaskArgValue::into_inner_value::( + task_arg.get(EXTRACT_ROOT).unwrap().clone(), + )); + let unpack_dest = + TaskArgValue::into_inner_value::(task_arg.get(UNPACK_DEST).unwrap().clone()); + let destination = extract_root.join(&unpack_dest); + + if !archive_path.exists() { + return Err(anyhow!(CmdErr::CopyTaskErr(format!( + "archive not found: {}", + archive_path.display() + )))); + } + + if extract_root.exists() { + fs::remove_dir_all(&extract_root)?; + } + fs::create_dir_all(&destination)?; + Self::unpack_archive(&archive_path, &destination)?; + + let mut result = HashMap::new(); + result.insert( + CMD.to_string(), + TaskArgValue::Str(format!("extract {}", archive_path.display())), + ); + result.insert(CMD_STATUS.to_string(), TaskArgValue::Number(0)); + result.insert( + CMD_OUTPUT.to_string(), + TaskArgValue::Str(destination.to_string_lossy().to_string()), + ); + Ok(Some(result)) + } +} diff --git a/src/cluster_mgr/src/cli/task/mod.rs b/src/cluster_mgr/src/cli/task/mod.rs index e9c3b192..66f3899a 100644 --- a/src/cluster_mgr/src/cli/task/mod.rs +++ b/src/cluster_mgr/src/cli/task/mod.rs @@ -7,7 +7,7 @@ pub mod config_fields; pub mod copy_task; pub mod db_update_log_task; pub mod db_update_task; -mod download_task; +pub mod download_task; mod eloq_bootstrap_task; pub mod eloq_dss_ctl_task; pub mod eloq_log_ctl_task; @@ -24,7 +24,8 @@ pub mod grpc_log; mod install_dep_pkg; mod local_backup_delete_task; mod local_copy_task; -mod monitor_ctl_task; +pub mod local_extract_task; +pub mod monitor_ctl_task; mod proxy_ctl_task; mod redis_op_task; pub mod rolling_upgrade; diff --git a/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs b/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs index 34832cd8..5c6b9b24 100644 --- a/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs +++ b/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs @@ -55,10 +55,10 @@ impl MonitorComponentCommand { format!(" {retention_flags}") }; format!( - r#"{home}/prometheus --web.listen-address="0.0.0.0:{port}" --storage.tsdb.path={home}/data{retention_flags} --config.file={home}/{PROMETHEUS_CONFIG_FILE} > /tmp/eloq_prometheus.log 2>&1 &"#, - home = home, - port = prom.port, - retention_flags = retention_flags, + r#"{home}/prometheus --web.listen-address="0.0.0.0:{port}" --web.enable-lifecycle --storage.tsdb.path={home}/data{retention_flags} --config.file={home}/{PROMETHEUS_CONFIG_FILE} > /tmp/eloq_prometheus.log 2>&1 &"#, + home = home, + port = prom.port, + retention_flags = retention_flags, ) }) } @@ -118,6 +118,7 @@ mod tests { .start(&monitor) .unwrap(); + assert!(cmd.contains("--web.enable-lifecycle")); assert!(cmd.contains(r#"--storage.tsdb.retention.time="30d""#)); assert!(cmd.contains(r#"--storage.tsdb.retention.size="50GB""#)); } diff --git a/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs b/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs index 79692dae..d1145f5c 100644 --- a/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs +++ b/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs @@ -7,9 +7,9 @@ use crate::cli::task::eloq_tx_ctl_task::{EloqTxCtlTask, ServerType}; use crate::cli::task::exec_custom_cmd::ExecCustomCommand; use crate::cli::task::failover_op_task::FailoverOpTask; use crate::cli::task::group::Config; +use crate::cli::task::local_extract_task::LocalExtractTask; use crate::cli::task::redis_op_task::{ClusterNodes, RedisOpTask}; use crate::cli::task::task_base::{TaskExecutionContext, TaskHost, TaskId, TaskInstance}; -use crate::cli::task::unpack_file_task::UnpackFileTask; use crate::cli::task::upload::upload_task_builder::{upload_tasks, UploadTaskBuilderType}; use crate::cli::SubCommand; use crate::config::config_base::DeployConfig; @@ -248,12 +248,14 @@ impl Step for DownloadAndUpload { )); let download_task = DownloadTask::instances(DownloadTask::from_urls(downloads)); - let barrier: Vec = [download_task.len(), upload_img.len()] + let extract_task = LocalExtractTask::from_config(&self.ctx.deploy)?; + let barrier: Vec = [download_task.len(), extract_task.len(), upload_img.len()] .into_iter() .filter(|&n| n > 0) .collect(); let mut executable = IndexMap::new(); executable.extend(download_task); + executable.extend(extract_task); executable.extend(upload_img); Ok(TaskExecutionContext { @@ -358,13 +360,11 @@ impl Step for StopLog { } } -pub struct UnpackTxLog { - ctx: UpgradeContext, -} +pub struct UnpackTxLog; impl UnpackTxLog { - pub fn new(ctx: UpgradeContext) -> Self { - Self { ctx } + pub fn new(_ctx: UpgradeContext) -> Self { + Self } } @@ -375,8 +375,7 @@ impl Step for UnpackTxLog { } async fn build(&self) -> anyhow::Result { - let unpack = UnpackFileTask::unpack_tx_and_log_nodes(&self.ctx.deploy); - Ok(single_barrier_ctx("unpack-tx-log", unpack)) + Ok(TaskExecutionContext::dummy()) } } @@ -605,13 +604,11 @@ impl Step for FailoverBackAndStopStandby { } } -pub struct UnpackStandby { - ctx: UpgradeContext, -} +pub struct UnpackStandby; impl UnpackStandby { - pub fn new(ctx: UpgradeContext) -> Self { - Self { ctx } + pub fn new(_ctx: UpgradeContext) -> Self { + Self } } @@ -622,11 +619,7 @@ impl Step for UnpackStandby { } async fn build(&self) -> anyhow::Result { - if !self.ctx.has_standby() { - return Ok(TaskExecutionContext::dummy()); - } - let unpack = UnpackFileTask::unpack_standby_nodes(&self.ctx.deploy); - Ok(single_barrier_ctx("unpack-standby", unpack)) + Ok(TaskExecutionContext::dummy()) } } diff --git a/src/cluster_mgr/src/cli/task/task_base.rs b/src/cluster_mgr/src/cli/task/task_base.rs index 90d07348..bdc3be68 100644 --- a/src/cluster_mgr/src/cli/task/task_base.rs +++ b/src/cluster_mgr/src/cli/task/task_base.rs @@ -491,4 +491,15 @@ impl TaskMgr { .run_all_tasks(tasks_execution, config, err_brk) .await } + + pub async fn run_context( + &'static self, + tasks_execution: TaskExecutionContext, + config: Config, + err_brk: bool, + ) -> anyhow::Result> { + self.task_controller + .run_all_tasks(tasks_execution, config, err_brk) + .await + } } diff --git a/src/cluster_mgr/src/cli/task/task_controller.rs b/src/cluster_mgr/src/cli/task/task_controller.rs index d2151036..a930043e 100644 --- a/src/cluster_mgr/src/cli/task/task_controller.rs +++ b/src/cluster_mgr/src/cli/task/task_controller.rs @@ -37,6 +37,65 @@ impl TaskController { Self { rx, tx } } + fn stage_label_from_tasks(task_group: &str, task_split: &[TaskInstance]) -> String { + if task_split.is_empty() { + return Self::stage_label(task_group); + } + + let task_ids = task_split + .iter() + .map(|task| task.task.identifier()) + .collect_vec(); + + if task_ids.iter().all(|id| id.cmd == "ssh-check") { + return "Check SSH connectivity".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "run_deps") { + return "Install runtime dependencies".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "check") { + return "Check port availability".to_string(); + } + if task_ids + .iter() + .all(|id| id.cmd == "deploy" && id.task == "mkdir") + { + return "Create remote directories".to_string(); + } + if task_ids + .iter() + .all(|id| id.cmd == "deploy" && id.task.ends_with("_download")) + { + return "Download release packages".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "extract") { + return "Extract release packages locally".to_string(); + } + if task_ids.iter().all(|id| { + matches!(id.cmd.as_str(), "deploy" | "update") + && (id.task.contains("upload") || id.task.starts_with("deploy_eloq_all_")) + }) { + return "Sync binaries and support files".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "config-update") { + return "Upload generated configuration".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "install") { + return "Bootstrap cluster metadata".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "start") { + return "Start services".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "topology") { + return "Refresh cluster topology".to_string(); + } + if task_ids.iter().all(|id| id.cmd == "monitor") { + return "Start monitor components".to_string(); + } + + Self::stage_label(task_group) + } + fn split_task(task_execution_context: &TaskExecutionContext) -> Vec<&'static [TaskInstance]> { let barrier = &task_execution_context.barrier; let task_install_vec = task_execution_context @@ -242,9 +301,10 @@ impl TaskController { task_names.join(", ") ); } else { + let stage_label = Self::stage_label_from_tasks(&task_group_string, task_split); println!( "[{stage_num}/{total_stages}] {} ({})", - Self::stage_label(&task_group_string), + stage_label, Self::stage_summary(task_split.len()) ); } diff --git a/src/cluster_mgr/src/cli/task/unpack_file_task.rs b/src/cluster_mgr/src/cli/task/unpack_file_task.rs index d9e274d1..df078ffd 100644 --- a/src/cluster_mgr/src/cli/task/unpack_file_task.rs +++ b/src/cluster_mgr/src/cli/task/unpack_file_task.rs @@ -227,7 +227,7 @@ impl UnpackFileTask { } tasks } - fn make_task_pair( + pub fn make_task_pair( config: &DeployConfig, host: &str, image: &str, diff --git a/src/cluster_mgr/src/cli/task/upload/data_dir_upload_builder.rs b/src/cluster_mgr/src/cli/task/upload/data_dir_upload_builder.rs index 77536f98..b69ead33 100644 --- a/src/cluster_mgr/src/cli/task/upload/data_dir_upload_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/data_dir_upload_builder.rs @@ -32,6 +32,7 @@ impl UploadTaskBuilder for DataDirUploadBuilder { extension: "datafarm".to_string(), host: host.to_string(), copy_dir: true, + delete_remote: true, }) .map(|upload_file| { build_task_instance( diff --git a/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs b/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs index 04cd097c..544ec3ac 100644 --- a/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs @@ -1,4 +1,5 @@ use crate::cli::task::group::Config; +use crate::cli::task::local_extract_task::LocalExtractTask; use crate::cli::task::task_base::{TaskId, TaskInstance}; use crate::cli::task::upload::upload_task_builder::{ build_task_instance, get_source_host, list_files_by_host, UploadTaskBuilder, @@ -15,8 +16,8 @@ use crate::config::{ }; use indexmap::IndexMap; use itertools::Itertools; -use std::collections::HashMap; use std::fs; +use std::path::Path; pub struct EloqUploadBuilder; @@ -55,21 +56,24 @@ impl EloqUploadBuilder { GRAFANA_FILE_KEY => config.get_host_list(DeploymentPackage::Grafana), _ => unreachable!(), }; - (dest_hosts, download_url) + (dest_hosts, download_url, file_key.clone()) }) - .filter(|(hosts, _url)| !hosts.is_empty()) - .flat_map(|(hosts, url)| { + .filter(|(hosts, _url, _file_key)| !hosts.is_empty()) + .flat_map(|(hosts, url, file_key)| { hosts .iter() .map(|host| { - let source = format!("{}/{}", url.cache_dir().unwrap(), url.file_name()); + let remote_home = Self::remote_home_for_key(config, &file_key); + let source = LocalExtractTask::staged_dir_for(url, &remote_home) + .to_string_lossy() + .to_string(); UploadFile { source, - dest: install_dir.clone(), - // fixme later - extension: "gz".to_string(), + dest: format!("{}/{}", install_dir, remote_home), + extension: "dir".to_string(), host: host.to_string(), - copy_dir: false, + copy_dir: true, + delete_remote: true, } }) .collect_vec() @@ -77,6 +81,17 @@ impl EloqUploadBuilder { .collect_vec() } + fn remote_home_for_key(config: &DeployConfig, file_key: &str) -> String { + match file_key { + ELOQ_FILE_KEY => config.deployment.product().home().to_string(), + ELOQ_LOG_FILE_KEY => "LogServer".to_string(), + PROMETHEUS_FILE_KEY => "prometheus".to_string(), + GRAFANA_FILE_KEY => "grafana".to_string(), + NODE_EXPORTER_FILE_KEY => "node_exporter".to_string(), + _ => unreachable!(), + } + } + fn build_eloq_misc_upload_file(&self, config: &DeployConfig) -> Vec { let mut all_files_path = Vec::new(); let log_start_path_opt = config.gen_log_start_script().unwrap(); @@ -150,19 +165,32 @@ impl EloqUploadBuilder { } } + let _ = all_files_path; + all_hosts_cloned .into_iter() - .map(|host| { - let source_files = list_files_by_host(&host, &config.deployment).join(" "); - UploadFile { - source: source_files, - dest: dest_file.clone(), - extension: "ini".to_string(), - host, - copy_dir: false, - } + .flat_map(|host| { + let dest_file = dest_file.clone(); + list_files_by_host(&host, &config.deployment) + .into_iter() + .map(move |source| { + let extension = Path::new(&source) + .extension() + .and_then(|ext| ext.to_str()) + .unwrap_or("upload") + .to_string(); + UploadFile { + source, + dest: dest_file.clone(), + extension, + host: host.clone(), + copy_dir: false, + delete_remote: false, + } + }) + .collect_vec() }) - .unique_by(|upload_file| upload_file.source.clone()) + .unique() .collect_vec() } } @@ -246,8 +274,11 @@ impl UploadTaskBuilder for EloqUploadBuilder { final_files .iter() .map(|upload_file| { - let extension = &upload_file.extension; - let task_name = format!("deploy_eloq_all_{extension}"); + let file_name = Path::new(&upload_file.source) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("upload"); + let task_name = format!("deploy_eloq_all_{}", file_name.replace('.', "_")); build_task_instance( source_host.clone(), upload_file.clone(), @@ -264,46 +295,16 @@ pub struct EloqUpload; impl EloqUpload { fn upload_group_by_dest(upload_files: Vec) -> Vec { - // Group the upload files by (host, dest) - let grouped: HashMap<(String, String), Vec> = upload_files - .into_iter() - .into_group_map_by(|upload| (upload.host.clone(), upload.dest.clone())); - - // Transform each group into a single UploadFile with aggregated fields - grouped - .into_iter() - .map(|((host, dest), group)| { - // Initialize vectors to collect sources and extensions - let mut sources = Vec::with_capacity(group.len()); - let mut extensions = Vec::with_capacity(group.len()); - - // Iterate once through the group to collect sources and extensions - for upload in group { - sources.push(upload.source); - extensions.push(upload.extension); - } - - // Join the collected sources and extensions - let aggregated_source = sources.join(" "); - let aggregated_extension = extensions.join(","); - - // Create a new UploadFile with aggregated data - UploadFile { - source: aggregated_source, - dest, - extension: aggregated_extension, - host, - copy_dir: false, - } - }) - .collect() + upload_files.into_iter().unique().collect_vec() } pub fn eloq_image_upload(config: &Deployment) -> Vec { - let install_dir = config.install_dir(); let img = config.tx_image(); let url = DownloadUrl::from_url_str(img).unwrap(); - let img_src = format!("{}/{}", url.cache_dir().unwrap(), url.file_name()); + let img_src = LocalExtractTask::staged_dir_for(&url, config.product().home()) + .to_string_lossy() + .to_string(); + let install_dir = format!("{}/{}", config.install_dir(), config.product().home()); let mut uploads = config .tx_service .tx_host_ports @@ -313,9 +314,10 @@ impl EloqUpload { UploadFile { source: img_src.clone(), dest: install_dir.clone(), - extension: "gz".to_string(), + extension: "dir".to_string(), host: host.to_string(), - copy_dir: false, + copy_dir: true, + delete_remote: false, } }) .collect_vec(); @@ -328,9 +330,10 @@ impl EloqUpload { UploadFile { source: img_src.clone(), dest: install_dir.clone(), - extension: "gz".to_string(), + extension: "dir".to_string(), host: host.to_string(), - copy_dir: false, + copy_dir: true, + delete_remote: false, } }) .collect_vec(); @@ -345,9 +348,10 @@ impl EloqUpload { UploadFile { source: img_src.clone(), dest: install_dir.clone(), - extension: "gz".to_string(), + extension: "dir".to_string(), host: host.to_string(), - copy_dir: false, + copy_dir: true, + delete_remote: false, } }) .collect_vec(); @@ -357,16 +361,20 @@ impl EloqUpload { if let Some(srv) = &config.log_service { let img = srv.image.as_ref().unwrap(); let url = DownloadUrl::from_url_str(img).unwrap(); - let img_src = format!("{}/{}", url.cache_dir().unwrap(), url.file_name()); + let img_src = LocalExtractTask::staged_dir_for(&url, "LogServer") + .to_string_lossy() + .to_string(); + let log_install_dir = format!("{}/LogServer", config.install_dir()); let ups = srv .log_host_unique() .iter() .map(|host| UploadFile { source: img_src.clone(), - dest: install_dir.clone(), - extension: "gz".to_string(), + dest: log_install_dir.clone(), + extension: "dir".to_string(), host: host.to_string(), - copy_dir: false, + copy_dir: true, + delete_remote: false, }) .collect_vec(); uploads.extend(ups); diff --git a/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs b/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs index 66938751..a5610e0b 100644 --- a/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs @@ -49,6 +49,7 @@ impl MonitorInfraConfUploadBuilder { extension: "json".to_string(), host: dest_host.to_string(), copy_dir: false, + delete_remote: false, }) } } @@ -124,6 +125,7 @@ impl MonitorInfraConfUploadBuilder { DeploymentPackage::Prometheus, ), copy_dir: false, + delete_remote: false, }; // Add the alert.rules file to the upload list @@ -139,6 +141,7 @@ impl MonitorInfraConfUploadBuilder { DeploymentPackage::Prometheus, ), copy_dir: false, + delete_remote: false, }; // Add the Prometheus configuration file to the upload list @@ -164,6 +167,7 @@ impl MonitorInfraConfUploadBuilder { DeploymentPackage::Grafana, ), copy_dir: false, + delete_remote: false, }; // Prepare the Grafana datasource UploadFile @@ -176,6 +180,7 @@ impl MonitorInfraConfUploadBuilder { DeploymentPackage::Grafana, ), copy_dir: false, + delete_remote: false, }; // Add the Grafana files to the upload list diff --git a/src/cluster_mgr/src/cli/task/upload/proxy_upload_builder.rs b/src/cluster_mgr/src/cli/task/upload/proxy_upload_builder.rs index 589f412f..c8373ad8 100644 --- a/src/cluster_mgr/src/cli/task/upload/proxy_upload_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/proxy_upload_builder.rs @@ -99,6 +99,7 @@ impl UploadTaskBuilder for ProxyUploadBuilder { extension: "".to_string(), // Empty because filename already includes extension host: host.clone(), copy_dir: false, + delete_remote: false, }; let (id, task) = build_task_instance( @@ -118,6 +119,7 @@ impl UploadTaskBuilder for ProxyUploadBuilder { extension: "".to_string(), host: host.clone(), copy_dir: false, + delete_remote: false, }; let (id, task) = build_task_instance( source_host.clone(), diff --git a/src/cluster_mgr/src/cli/task/upload/tx_conf_upload_builder.rs b/src/cluster_mgr/src/cli/task/upload/tx_conf_upload_builder.rs index 40169fac..7466673e 100644 --- a/src/cluster_mgr/src/cli/task/upload/tx_conf_upload_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/tx_conf_upload_builder.rs @@ -116,6 +116,7 @@ impl UploadTaskBuilder for TxConfUpload { extension: "ini".to_string(), host: host.to_string(), copy_dir: false, + delete_remote: false, }); } @@ -150,6 +151,7 @@ impl UploadTaskBuilder for TxConfUpload { extension: "ini".to_string(), host: host.to_string(), copy_dir: false, + delete_remote: false, }); } continue; @@ -204,6 +206,7 @@ impl UploadTaskBuilder for TxConfUpload { extension: "ini".to_string(), host: host.to_string(), copy_dir: false, + delete_remote: false, }); } } @@ -231,6 +234,7 @@ impl UploadTaskBuilder for TxConfUpload { extension: "ini".to_string(), host: host.to_string(), copy_dir: false, + delete_remote: false, }) }) .collect_vec(); @@ -302,6 +306,7 @@ impl TxConfUpload { extension: "tls".to_string(), host: host.to_string(), copy_dir: false, + delete_remote: false, }) }) .collect::>() @@ -416,6 +421,7 @@ impl TxConfUpload { extension: "ini".to_string(), host: host.to_string(), copy_dir: false, + delete_remote: false, }); } continue; @@ -476,6 +482,7 @@ impl TxConfUpload { extension: "ini".to_string(), host: host.clone(), copy_dir: false, + delete_remote: false, }); } } @@ -568,6 +575,7 @@ impl TxConfUpload { extension: "ini".to_string(), host: host.clone(), copy_dir: false, + delete_remote: false, }; let (id, instance) = build_task_instance( @@ -644,6 +652,7 @@ impl TxConfUpload { extension: "".to_string(), // No extension for this file host: host.clone(), copy_dir: false, + delete_remote: false, }; let (task_id, task_instance) = build_task_instance( diff --git a/src/cluster_mgr/src/cli/task/upload/upload_task.rs b/src/cluster_mgr/src/cli/task/upload/upload_task.rs index fe9f9bc1..d9a2fa66 100644 --- a/src/cluster_mgr/src/cli/task/upload/upload_task.rs +++ b/src/cluster_mgr/src/cli/task/upload/upload_task.rs @@ -1,7 +1,7 @@ use crate::cli::task::task_base::{ CmdErr, ExecutionValue, TaskArgValue, TaskExecutor, TaskHost, TaskId, }; -use crate::cli::task::upload::upload_task_builder::{SCP_ARGS, SCP_COMMAND}; +use crate::cli::task::upload::upload_task_builder::{TRANSFER_ARGS, TRANSFER_COMMAND}; use crate::cli::{CMD, CMD_OUTPUT, CMD_STATUS}; use crate::task_return_value; use anyhow::anyhow; @@ -33,29 +33,29 @@ impl TaskExecutor for UploadTask { task_arg: HashMap, ) -> anyhow::Result> { info!("execute {}", self.task_id.format_string()); - let scp = task_arg - .get(SCP_COMMAND) + let sync = task_arg + .get(TRANSFER_COMMAND) .map(|value| value.to_string()) - .unwrap_or_else(|| "scp".to_string()); - let scp_args = task_arg - .get(SCP_ARGS) + .unwrap_or_else(|| "rsync".to_string()); + let sync_args = task_arg + .get(TRANSFER_ARGS) .cloned() .map(TaskArgValue::into_inner_value::>) .ok_or_else(|| { anyhow!(CmdErr::UploadErr( - scp.clone(), - "missing scp args".to_string() + sync.clone(), + "missing sync args".to_string() )) })?; - info!("Running local scp: {}", scp); + info!("Running local sync: {}", sync); let output = timeout( Duration::from_secs(120), - Command::new("scp").args(&scp_args).output(), + Command::new(&sync).args(&sync_args).output(), ) .await - .map_err(|_| anyhow!(CmdErr::UploadErr(scp.clone(), "timed out".to_string())))? - .map_err(|e| anyhow!(CmdErr::UploadErr(scp.clone(), e.to_string())))?; + .map_err(|_| anyhow!(CmdErr::UploadErr(sync.clone(), "timed out".to_string())))? + .map_err(|e| anyhow!(CmdErr::UploadErr(sync.clone(), e.to_string())))?; let code = output.status.code().unwrap_or(-1); let command_output = format!( "{}{}", @@ -68,7 +68,7 @@ impl TaskExecutor for UploadTask { format!("{}: {}", code, command_output.trim()) }; let mut result = std::collections::HashMap::new(); - result.insert(CMD.to_string(), TaskArgValue::Str(scp.clone())); + result.insert(CMD.to_string(), TaskArgValue::Str(sync.clone())); result.insert(CMD_STATUS.to_string(), TaskArgValue::Number(code)); result.insert(CMD_OUTPUT.to_string(), TaskArgValue::Str(command_output)); task_return_value!( @@ -79,7 +79,7 @@ impl TaskExecutor for UploadTask { } else { status_code.to_string() }; - CmdErr::UploadErr(scp.clone(), detail) + CmdErr::UploadErr(sync.clone(), detail) }, "UploadTask" ) diff --git a/src/cluster_mgr/src/cli/task/upload/upload_task_builder.rs b/src/cluster_mgr/src/cli/task/upload/upload_task_builder.rs index fab9490c..b9b9b186 100644 --- a/src/cluster_mgr/src/cli/task/upload/upload_task_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/upload_task_builder.rs @@ -31,8 +31,8 @@ pub trait UploadTaskBuilder { fn build(&self, config: &Config) -> IndexMap; } -pub(crate) const SCP_COMMAND: &str = "_scp_cmd_"; -pub(crate) const SCP_ARGS: &str = "_scp_args_"; +pub(crate) const TRANSFER_COMMAND: &str = "_transfer_cmd_"; +pub(crate) const TRANSFER_ARGS: &str = "_transfer_args_"; pub(crate) const SOURCE_IP: &str = "_source_ip_"; pub(crate) fn scp_args(upload_file: &UploadFile, conn: Connection) -> Vec { @@ -69,6 +69,39 @@ pub(crate) fn scp_args(upload_file: &UploadFile, conn: Connection) -> Vec Vec { + let auth_key = conn.ssh_auth_key().unwrap(); + let (remote_host, port) = conn.ssh_endpoint(&upload_file.host); + let ssh_cmd = format!( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o BatchMode=yes -o ConnectTimeout=10 -o PreferredAuthentications=publickey -i {} -p {}", + auth_key, port + ); + let mut args = vec![ + "-az".to_string(), + "-e".to_string(), + ssh_cmd, + "--rsync-path".to_string(), + format!("mkdir -p {} && rsync", upload_file.dest.trim()), + "--partial".to_string(), + "--progress".to_string(), + ]; + if upload_file.delete_remote { + args.push("--delete".to_string()); + } + let mut source = upload_file.source.trim().to_string(); + if upload_file.copy_dir && !source.ends_with('/') { + source.push('/'); + } + args.push(source); + args.push(format!( + "{}@{}:{}", + conn.username, + remote_host, + upload_file.dest.trim() + )); + args +} + #[derive(Clone, Debug)] pub enum UploadTaskBuilderType { DataDir, @@ -200,15 +233,21 @@ pub(crate) fn build_task_instance( }; let conn = config.conn_ref(); - let scp_args = scp_args(&upload_file, conn.clone()); - let scp_cmd = format!("scp {}", scp_args.join(" ")); + let (transfer_bin, transfer_args) = if upload_file.copy_dir { + ("rsync".to_string(), sync_args(&upload_file, conn.clone())) + } else { + ("scp".to_string(), scp_args(&upload_file, conn.clone())) + }; let upload_task = UploadTask::new(task_id.clone()); ( task_id, TaskInstance { task_input: HashMap::from([ - (SCP_COMMAND.to_string(), TaskArgValue::Str(scp_cmd)), - (SCP_ARGS.to_string(), TaskArgValue::List(scp_args)), + ( + TRANSFER_COMMAND.to_string(), + TaskArgValue::Str(transfer_bin), + ), + (TRANSFER_ARGS.to_string(), TaskArgValue::List(transfer_args)), (SOURCE_IP.to_string(), TaskArgValue::Str(source_host)), ]), task: Box::new(upload_task), diff --git a/src/cluster_mgr/src/config/config_base.rs b/src/cluster_mgr/src/config/config_base.rs index 95bc3248..cde1e150 100644 --- a/src/cluster_mgr/src/config/config_base.rs +++ b/src/cluster_mgr/src/config/config_base.rs @@ -68,6 +68,7 @@ pub struct UploadFile { pub extension: String, pub host: String, pub copy_dir: bool, + pub delete_remote: bool, } #[serde_with::skip_serializing_none] @@ -421,7 +422,9 @@ impl DeployConfig { let bin = self.deployment.client_bin(); let host_port = self.deployment.tx_service.tx_host_ports.first().unwrap(); let parts: Vec<&str> = host_port.split(':').collect(); - format!("{bin} -h {} -p {}", parts[0], parts[1]) + let port = parts[1].parse::().unwrap_or(6379); + let (host, port) = self.service_endpoint(parts[0], port); + format!("{bin} -h {host} -p {port}") } pub fn product(&self) -> Product { diff --git a/src/cluster_mgr/src/config/deployment.rs b/src/cluster_mgr/src/config/deployment.rs index 532801d4..66200df6 100644 --- a/src/cluster_mgr/src/config/deployment.rs +++ b/src/cluster_mgr/src/config/deployment.rs @@ -173,6 +173,8 @@ impl Product { pub struct Hardware { pub cpu: u16, pub memory: u32, // MiB + #[serde(skip_serializing_if = "Option::is_none")] + pub event_dispatcher_num: Option, } pub enum Version { @@ -1489,7 +1491,7 @@ impl Deployment { let key = "event_dispatcher_num"; let val = set_by_user!(ini.get(SECTION_LOCAL, key), u16); if val.is_none() { - let core_io = core_tx.div_ceil(8); + let core_io = hw.event_dispatcher_num.unwrap_or(core_tx.div_ceil(8)); ini.set(SECTION_LOCAL, key, Some(core_io.to_string())); } diff --git a/src/cluster_mgr/src/config/mod.rs b/src/cluster_mgr/src/config/mod.rs index a7509d88..44d406b4 100644 --- a/src/cluster_mgr/src/config/mod.rs +++ b/src/cluster_mgr/src/config/mod.rs @@ -40,7 +40,6 @@ pub const GRAFANA_PROMETHEUS_DS_FILE: &str = "prometheus-datasource.yml"; pub const GRAFANA_CONFIG_TEMPLATE: &str = "grafana_config.ini"; pub const GRAFANA_CONFIG_FILE: &str = "defaults.ini"; -pub const CLOUDFRONT: &str = "download.eloqdata.com"; pub const CDN: &str = "https://download.eloqdata.com"; #[macro_export] @@ -116,12 +115,22 @@ impl DownloadUrl { match self { DownloadUrl::Local(_) => {} DownloadUrl::Remote(url) => { - if url.domain() == Some(CLOUDFRONT) { + if url.domain() == Some("download.eloqdata.com") { let mut seg = url.path_segments().unwrap(); let _filename = seg.next_back(); for d in seg { dir.push(d); } + } else if url.domain() == Some("github.com") { + let file_name = self.file_name(); + if let Some(parsed) = crate::github_release::parse_asset_name(&file_name) { + let product_dir = match parsed.product.as_str() { + "log-service" => "logservice", + other => other, + }; + dir.push(product_dir); + dir.push(parsed.store); + } } } } diff --git a/src/cluster_mgr/src/github_release.rs b/src/cluster_mgr/src/github_release.rs new file mode 100644 index 00000000..ed70021a --- /dev/null +++ b/src/cluster_mgr/src/github_release.rs @@ -0,0 +1,241 @@ +use crate::config::config_base::VersionRow; +use crate::config::deployment::{version_digits, Product}; +use anyhow::{anyhow, bail, Result}; +use serde::Deserialize; +use std::collections::BTreeSet; + +const GITHUB_RELEASES_API: &str = + "https://api.github.com/repos/eloqdata/eloqkv/releases?per_page=100"; +const ELOQKV_REPO: &str = "eloqdata/eloqkv"; + +#[derive(Debug, Clone, Deserialize)] +pub struct GitHubRelease { + pub tag_name: String, + pub draft: bool, + pub prerelease: bool, + #[serde(default)] + pub assets: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GitHubAsset { + pub name: String, + pub browser_download_url: String, + pub digest: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedAssetName { + pub product: String, + pub version: String, + pub store: String, + pub os: String, + pub arch: String, +} + +#[derive(Debug, Clone)] +pub struct ReleaseAssetMatch { + pub version: String, + pub url: String, + pub digest: Option, +} + +pub async fn fetch_eloqkv_releases(client: &reqwest::Client) -> Result> { + let response = client + .get(GITHUB_RELEASES_API) + .header(reqwest::header::USER_AGENT, "eloqctl") + .header(reqwest::header::ACCEPT, "application/vnd.github+json") + .send() + .await?; + + if !response.status().is_success() { + bail!("fetch GitHub releases failed: {}", response.status()); + } + + let releases = response.json::>().await?; + Ok(releases + .into_iter() + .filter(|release| !release.draft && !release.prerelease) + .collect()) +} + +pub fn list_versions_from_releases( + releases: &[GitHubRelease], + product: Option, + store: Option<&str>, +) -> Vec { + let requested_product = product.unwrap_or(Product::EloqKV); + let requested_product_name = requested_product.name(); + let mut rows = BTreeSet::new(); + + for release in releases { + let release_version = normalize_tag(&release.tag_name); + if version_digits(&release_version).is_err() { + continue; + } + + for asset in &release.assets { + let Some(parsed) = parse_asset_name(&asset.name) else { + continue; + }; + if parsed.product != requested_product_name { + continue; + } + if parsed.version != release_version { + continue; + } + if store.is_some_and(|requested| requested != parsed.store) { + continue; + } + + rows.insert((parsed.product, parsed.store, parsed.version)); + } + } + + rows.into_iter() + .map(|(product, store, version)| VersionRow { + product, + store, + version, + }) + .collect() +} + +pub fn find_product_asset( + releases: &[GitHubRelease], + product: &str, + version: &str, + store: &str, + os: &str, + arch: &str, +) -> Result { + let normalized_version = normalize_tag(version); + + for release in releases { + if normalize_tag(&release.tag_name) != normalized_version { + continue; + } + + for asset in &release.assets { + let Some(parsed) = parse_asset_name(&asset.name) else { + continue; + }; + if parsed.product == product + && parsed.version == normalized_version + && parsed.store == store + && parsed.os == os + && parsed.arch == arch + { + return Ok(ReleaseAssetMatch { + version: normalized_version, + url: asset.browser_download_url.clone(), + digest: asset.digest.clone(), + }); + } + } + } + + Err(anyhow!( + "no GitHub release asset found for product={}, version={}, store={}, os={}, arch={} in {}", + product, + normalized_version, + store, + os, + arch, + ELOQKV_REPO + )) +} + +pub fn find_eloqkv_asset( + releases: &[GitHubRelease], + product: Product, + version: &str, + store: &str, + os: &str, + arch: &str, +) -> Result { + find_product_asset(releases, product.name(), version, store, os, arch) +} + +pub fn normalize_tag(tag: &str) -> String { + tag.trim_start_matches('v').to_string() +} + +pub fn parse_asset_name(name: &str) -> Option { + let stem = name.strip_suffix(".tar.gz")?; + let parts = stem.split('-').collect::>(); + if parts.len() < 5 { + return None; + } + + let product_len = match parts.first().copied() { + Some("eloqkv") => 1, + Some("log") if parts.get(1).copied() == Some("service") => 2, + _ => return None, + }; + let product = parts[..product_len].join("-"); + let version = parts.get(product_len)?.to_string(); + let os = parts.get(parts.len() - 2)?.to_string(); + let arch = parts.last()?.to_string(); + let store = parts[product_len + 1..parts.len() - 2].join("-"); + if store.is_empty() { + return None; + } + + Some(ParsedAssetName { + product, + version, + store, + os, + arch, + }) +} + +#[cfg(test)] +mod tests { + use super::{list_versions_from_releases, parse_asset_name, GitHubAsset, GitHubRelease}; + use crate::config::deployment::Product; + + #[test] + fn parse_asset_name_handles_rocks_s3() { + let asset = parse_asset_name("eloqkv-1.2.2-rocks_s3-ubuntu24-amd64.tar.gz").unwrap(); + assert_eq!(asset.product, "eloqkv"); + assert_eq!(asset.version, "1.2.2"); + assert_eq!(asset.store, "rocks_s3"); + assert_eq!(asset.os, "ubuntu24"); + assert_eq!(asset.arch, "amd64"); + } + + #[test] + fn parse_asset_name_handles_log_service() { + let asset = parse_asset_name("log-service-1.2.2-rocksdb-ubuntu24-amd64.tar.gz").unwrap(); + assert_eq!(asset.product, "log-service"); + assert_eq!(asset.store, "rocksdb"); + } + + #[test] + fn list_versions_aggregates_assets() { + let releases = vec![GitHubRelease { + tag_name: "1.2.2".to_string(), + draft: false, + prerelease: false, + assets: vec![ + GitHubAsset { + name: "eloqkv-1.2.2-rocksdb-ubuntu24-amd64.tar.gz".to_string(), + browser_download_url: "https://example.invalid/rocksdb".to_string(), + digest: Some("sha256:abc".to_string()), + }, + GitHubAsset { + name: "eloqkv-1.2.2-rocks_s3-ubuntu24-amd64.tar.gz".to_string(), + browser_download_url: "https://example.invalid/rocks_s3".to_string(), + digest: Some("sha256:def".to_string()), + }, + ], + }]; + + let rows = list_versions_from_releases(&releases, Some(Product::EloqKV), None); + assert_eq!(rows.len(), 2); + assert!(rows.iter().any(|row| row.store == "rocksdb")); + assert!(rows.iter().any(|row| row.store == "rocks_s3")); + } +} diff --git a/src/cluster_mgr/src/lib.rs b/src/cluster_mgr/src/lib.rs index ed045002..9d91c826 100644 --- a/src/cluster_mgr/src/lib.rs +++ b/src/cluster_mgr/src/lib.rs @@ -14,6 +14,7 @@ use strum_macros::AsRefStr; pub mod cli; pub mod config; +pub mod github_release; pub mod state; #[macro_export] diff --git a/tests/README.md b/tests/README.md index 2fb3533e..220ea396 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,132 +1,233 @@ # E2E Tests -A single script (`tests/e2e/test.sh`) starts one Docker environment, deploys one EloqKV cluster, then runs all scenarios sequentially against it. The whole suite completes in ~5 minutes. +Deploys an EloqKV cluster in Docker containers via `eloqctl launch`, then runs +multi-SDK stress workloads against it. Every test uses both standalone and cluster +Redis clients with TLS enabled. -## Architecture +## Quick Start +```sh +cd /home/starrysky/workspace/eloqdata-kernel/eloq_waiter + +# Build eloqctl (one time) +scripts/install-dev.sh + +# Run everything: launch → Python → Go → TS → remove +bash tests/e2e/cmd_stress_test.sh ``` -tests/ -├── e2e/ -│ ├── test.sh ← single test entry point -│ └── topology.yaml ← topology for the e2e cluster -├── docker_ha/ ← shared Docker infrastructure -│ ├── Dockerfile ← Ubuntu 24.04 + SSH server -│ ├── docker-compose.yaml ← 3-node bridge network (172.28.10.x) -│ ├── topology.yaml ← topology template (referenced by docker_env.sh) -│ ├── id_ed25519 ← SSH key pair for eloq user -│ └── authorized_keys -└── docker_env.sh ← shared shell helpers + +The default stress duration is now `300` seconds per SDK step. The appended +high-concurrency `INFO` burst in each SDK step also defaults to `300` seconds. + +## Run specific steps + +```sh +# Launch only +STEPS=launch bash tests/e2e/cmd_stress_test.sh + +# Stress only (against already-running cluster) +STEPS=py-stress,go-stress,ts-stress bash tests/e2e/cmd_stress_test.sh + +# Override the default 300-second stress duration +DURATION_SECONDS=120 INFO_ONLY_DURATION_SECONDS=120 \ + STEPS=py-stress,go-stress,ts-stress bash tests/e2e/cmd_stress_test.sh ``` -## How It Works +## Control Node Usage -1. **Docker env starts once** — `docker_env.sh` builds an Ubuntu image, starts 3 containers on a bridge network, waits for SSH -2. **Cluster deploys once** — `eloqctl launch` installs deps, deploys EloqKV, boots one tx + one standby + one voter -3. **All scenarios run sequentially** — every test scenario operates on the same running cluster: - - status, versions, list, export, connect - - rolling update via `plan` + `apply` - - scale add / remove - - stop → check topology → start - - failover (promote standby to master) - - monitor status, log-service status - - exec (remote shell command) - - schema upgrade - - remove (cleanup verification) -4. **Docker env destroyed once** — trap cleans up on exit or failure +The Docker E2E environment uses `eloq-node-4` as the control node. Install the +current host-built `eloqctl` into that container with: + +```sh +tests/install_control_eloqctl.sh +``` -## Prerequisites +Then log in directly: ```sh -# Build and install the dev eloqctl binary -scripts/install-dev.sh +ssh -i "tests/docker_ha/id_ed25519" -p 2224 eloq@127.0.0.1 ``` -Requires Docker and Rust toolchain (see repo README). +Inside `node4`, `eloqctl` is installed at `/usr/local/bin/eloqctl` and uses +`/home/eloq/.eloqctl` as `ELOQCTL_HOME`. + +### Generate launch topology inside `node4` -## Running +The template topology lives at `tests/e2e/topology.yaml`. From a shell inside +`node4`, generate the control-node-ready topology and launch from it: ```sh -# Single script — runs everything -bash tests/e2e/test.sh +cd /workspace/eloq_waiter +export DOCKER_E2E_DIR=/workspace/eloq_waiter/tests/docker_ha +export REPO_ROOT=/workspace/eloq_waiter +source tests/docker_env.sh +render_topology_for_control tests/e2e/topology.yaml tests/e2e/topology.generated.yaml +eloqctl launch --skip-deps tests/e2e/topology.generated.yaml +``` -# Full push gate (includes format, build, clippy, E2E) -scripts/test-before-push.sh +To use a different EloqKV version, set `ELOQKV_VERSION` before sourcing: -# Install git pre-push hook (auto-runs test-before-push.sh) -scripts/install-git-hooks.sh +```sh +export ELOQKV_VERSION=1.3.0 ``` -Override the binary path: +### Check status and connect + +After launch, `eloqctl status ` prints both the status table and a +ready-to-copy `redis-cli` command: + ```sh -ELOQCTL=/custom/path/to/cluster_mgr bash tests/e2e/test.sh +eloqctl status test-e2e +``` + +## Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `STEPS` | `launch,py-stress,go-stress,ts-stress,remove` | Comma-separated steps | +| `WORKERS` | `16` | Total workers (split evenly: standalone / cluster client) | +| `DURATION_SECONDS` | `300` | Main stress duration per SDK step | +| `INFO_ONLY_DURATION_SECONDS` | `300` | Duration of the appended high-concurrency `INFO` burst | +| `INFO_ONLY_WORKERS` | `64` | Worker count for the appended `INFO` burst | +| `INFO_ONLY_INFLIGHT` | `16` | Inflight slots per worker for the appended `INFO` burst | +| `INFO_ONLY_REPEAT` | `50` | Repeat count per round for the appended `INFO` burst | +| `KEY_COUNT` | `256` | Preloaded key count | +| `CMD_TIMEOUT` | `5` | Per-command timeout (seconds) | +| `PROGRESS_INTERVAL` | `5` | Progress report interval (seconds) | +| `TLS_ENABLED` | `1` | Enable TLS on cluster and clients | +| `SKIP_DEPS` | `1` | Skip OS dep installation on nodes | + +## Directory Layout + +``` +tests/ +├── README.md +├── docker_env.sh # shared helpers: Docker Compose, SSH, MinIO +├── docker_ha/ +│ ├── docker-compose.yaml # 4-node Ubuntu + MinIO + stress containers +│ ├── Dockerfile # SSH image for eloq nodes +│ ├── Dockerfile.stress # Python 3.13 + redis-py +│ ├── Dockerfile.stress_go # Go 1.24 + go-redis/v9 +│ ├── Dockerfile.stress_ts # Node 22 + ioredis +│ ├── id_ed25519 / id_ed25519.pub # auto-generated SSH key +│ └── authorized_keys +└── e2e/ + ├── cmd_stress_test.sh # main entry point ★ + ├── cmd_stress_py/ + │ ├── main.py # Python full-command stress client + │ └── requirements.txt # pinned redis-py dependency (5.0.1) + ├── cmd_stress_go/ + │ ├── main.go # Go full-command stress client + │ ├── go.mod / go.sum + ├── cmd_stress_ts/ + │ ├── main.ts # TypeScript full-command stress client + │ ├── package.json / package-lock.json / tsconfig.json + └── topology.yaml # cluster topology template ``` ## Command Coverage -### Covered by E2E - -| Command | How tested | -|---------|-----------| -| `launch` | Full cluster deploy with deps, config, bootstrap, start | -| `status --wait` | After every mutation: verifies serving master + process health | -| `versions` | Lists available versions, output contains version numbers | -| `list` | Registered clusters include the test cluster | -| `export` | Exported YAML contains cluster_name | -| `connect` | Client command string exits successfully | -| `plan` + `apply` | Rolling update via checkpoint_interval change | -| `scale --add-nodes` | Adds new standby, verifies cluster stays healthy | -| `scale --remove-nodes` | Removes old standby, verifies cluster stays healthy | -| `stop + start` | Full stop/start cycle with intermediate `check` validation | -| `check` | Topology validation on stopped cluster (ports free etc.) | -| `failover` | Promotes standby to master, verifies cluster recovers | -| `monitor status` | Reports correctly even without monitor configured | -| `log-service status` | Reports correctly even without log-service configured | -| `exec` | Runs remote shell command on cluster hosts | -| `upgrade` | Schema upgrade exits successfully | -| `remove --force` | Cluster removed, no longer present in `list` | - -### Not Covered - -| Command | Reason | -|---------|--------| -| `deploy`, `install`, `run-deps` | Tested implicitly via `launch` | -| `restart` | Equivalent to `stop` + `start`, covered separately | -| `scalelog` | Requires `log_service` config with log binaries | -| `backup start/list/remove` | SSH endpoint mapping bug in `--dest-node` path | -| `update` | Requires version download from remote PostgreSQL | -| `proxy` | Legacy product, not active | -| `demo` | Interactive, not suitable for automated testing | -| `completion` | Shell generation, no runtime state to verify | - -## Test Output - -Passing output looks like: -``` -[1/6] Launch cluster ... OK -[2/6] Verify cluster status ... OK -[3/6] Test read-only commands ... OK -[4/6] Rolling update ... OK -[5/6] Scale add standby ... OK -[6/6] Scale remove old standby ... OK -[7/14] Stop cluster and validate topology ... OK -[8/14] Restart cluster ... OK -[9/14] Failover standby to master ... OK -[10/14] Monitor and log-service status ... OK -[11/14] Exec custom remote command ... OK -[12/14] Schema upgrade ... OK -[13/14] Remove cluster ... OK -PASS: all E2E tests completed -``` - -## Adding New Tests - -Add a new step to `tests/e2e/test.sh` following the existing pattern: +Each SDK stress test covers **104 Redis commands** across all families +(string, hash, list, set, sorted-set, generic/key, server/connection). + +Every test runs **half the workers with a standalone client** (direct to master) +and **half with a cluster-aware client** (auto slot routing). Results for both +modes are reported separately. The standalone client always targets the master. +Only the cluster-aware client is configured to read from replicas. + +After the normal mixed-command phase, each SDK appends a dedicated, +high-concurrency `INFO` burst. + +TLS is enabled by default with self-signed certs (`rejectUnauthorized: false` / +`ssl_cert_reqs=CERT_NONE` / `InsecureSkipVerify`). + +## Troubleshooting + +If launch fails, rebuild Docker images: + +```sh +cd tests/docker_ha && docker compose build --no-cache +``` + +Check cluster health: + ```sh -echo "[N/TOTAL] Your test description" -"${ELOQCTL}" your-command ... > "${SCRIPT_DIR}/your-command.log" 2>&1 || { - echo "FAIL: your test" - cat "${SCRIPT_DIR}/your-command.log" - exit 1 -} -echo " OK" +eloqctl status test-e2e --wait 30 +``` + +Logs auto-clean unless `KEEP_LOGS=1` is set. + +## Connecting from Host + +The cluster nodes are exposed on localhost ports. After `docker compose up` and +a successful `eloqctl launch`: + +| Node | Host Port | Docker IP | +|------|-----------|-----------| +| node-1 | `127.0.0.1:16371` | `172.28.10.11:6379` | +| node-2 | `127.0.0.1:16372` | `172.28.10.12:6379` | +| node-3 | `127.0.0.1:16373` | `172.28.10.13:6379` | + +### Check cluster topology + +```sh +redis-cli -h 127.0.0.1 -p 16371 --tls --insecure -a testpass CLUSTER NODES +``` + +The output shows which node is `master` and which is `slave`: + +``` + 172.28.10.11:6379@16380 myself,slave ... + 172.28.10.12:6379@16380 master ... +``` + +### Connect with redis-py (Python) + +```python +import ssl, redis +TLS = {'ssl': True, 'ssl_cert_reqs': ssl.CERT_NONE, 'ssl_check_hostname': False} + +# standalone — connect directly to one node +r = redis.Redis(host='127.0.0.1', port=16371, password='testpass', **TLS) + +# cluster mode — auto-routes to correct node +from redis.cluster import RedisCluster, ClusterNode +rc = RedisCluster(startup_nodes=[ + ClusterNode('127.0.0.1', 16371), + ClusterNode('127.0.0.1', 16372), +], password='testpass', **TLS) +``` + +### Connect with go-redis (Go) + +```go +// standalone +c := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:16371", Password: "testpass", + TLSConfig: &tls.Config{InsecureSkipVerify: true}, +}) + +// cluster +cc := redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{"127.0.0.1:16371", "127.0.0.1:16372"}, + Password: "testpass", + TLSConfig: &tls.Config{InsecureSkipVerify: true}, +}) +``` + +### Connect with ioredis (TypeScript) + +```typescript +// standalone +const r = new Redis({ + host: "127.0.0.1", port: 16371, password: "testpass", + tls: { rejectUnauthorized: false }, +}); + +// cluster +const c = new Cluster([ + { host: "127.0.0.1", port: 16371 }, + { host: "127.0.0.1", port: 16372 }, +], { redisOptions: { password: "testpass", + tls: { rejectUnauthorized: false } } }); ``` diff --git a/tests/docker_env.sh b/tests/docker_env.sh index 194bbc1b..54b98c11 100755 --- a/tests/docker_env.sh +++ b/tests/docker_env.sh @@ -7,14 +7,34 @@ REPO_ROOT="${REPO_ROOT:-$(cd "${DOCKER_E2E_DIR}/../.." && pwd)}" export ELOQCTL_HOME="${ELOQCTL_HOME:-${HOME}/.eloqctl}" ELOQCTL="${ELOQCTL:-${ELOQCTL_HOME}/bin/cluster_mgr}" ELOQCTL_DOCKER_SSH_KEY="${ELOQCTL_DOCKER_SSH_KEY:-${DOCKER_E2E_DIR}/id_ed25519}" +ELOQCTL_CONTROL_SSH_KEY="${ELOQCTL_CONTROL_SSH_KEY:-/home/eloq/.ssh/id_ed25519}" +CONTROL_NODE_SERVICE="${CONTROL_NODE_SERVICE:-eloq-node-4}" +CONTROL_REPO_ROOT="${CONTROL_REPO_ROOT:-/workspace/eloq_waiter}" +CONTROL_ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME:-/home/eloq/.eloqctl}" +CONTROL_ELOQCTL="${CONTROL_ELOQCTL:-${CONTROL_ELOQCTL_HOME}/bin/cluster_mgr}" +HOST_ELOQCTL_HOME="${HOST_ELOQCTL_HOME:-${REPO_ROOT}/.tmp-eloqctl}" export ELOQCTL_DOCKER_SSH_KEY CLEANUP_TIMEOUT_SECONDS="${CLEANUP_TIMEOUT_SECONDS:-20}" +ELOQKV_VERSION="${ELOQKV_VERSION:-1.2.2}" +MINIO_ALIAS="${MINIO_ALIAS:-e2e-minio}" +MINIO_ENDPOINT="${MINIO_ENDPOINT:-http://127.0.0.1:19000}" +MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}" +MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minioadmin}" +MINIO_BUCKET="${MINIO_BUCKET:-storeeloqservice}" + +COMPOSE_BASE="${DOCKER_E2E_DIR}/docker-compose.yaml" +COMPOSE_OVERRIDE="" +if [ -n "${COMPOSE_OVERRIDE_FILE:-}" ] && [ -f "${COMPOSE_OVERRIDE_FILE}" ]; then + COMPOSE_OVERRIDE="${COMPOSE_OVERRIDE_FILE}" +fi compose() { + local args=(-f "${COMPOSE_BASE}") + [ -n "${COMPOSE_OVERRIDE}" ] && args+=(-f "${COMPOSE_OVERRIDE}") if docker compose version >/dev/null 2>&1; then - docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" "$@" + docker compose "${args[@]}" "$@" else - docker-compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" "$@" + docker-compose "${args[@]}" "$@" fi } @@ -54,7 +74,49 @@ ensure_ssh_key() { render_topology() { local source_topology="$1" local rendered_topology="$2" - sed "s|\${ELOQCTL_DOCKER_SSH_KEY}|${ELOQCTL_DOCKER_SSH_KEY}|g" "${source_topology}" > "${rendered_topology}" + sed -e "s|\${ELOQCTL_DOCKER_SSH_KEY}|${ELOQCTL_DOCKER_SSH_KEY}|g" \ + -e "s|\${ELOQKV_VERSION}|${ELOQKV_VERSION}|g" \ + "${source_topology}" > "${rendered_topology}" +} + +render_topology_for_control() { + local source_topology="$1" + local rendered_topology="$2" + sed -e "s|\${ELOQCTL_DOCKER_SSH_KEY}|${ELOQCTL_CONTROL_SSH_KEY}|g" \ + -e "s|\${ELOQKV_VERSION}|${ELOQKV_VERSION}|g" \ + "${source_topology}" > "${rendered_topology}" +} + +control_exec() { + compose exec -T -u eloq "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq "$@" +} + +prepare_control_node() { + compose exec -T "${CONTROL_NODE_SERVICE}" bash -lc " + set -eu + install -d -m 700 -o eloq -g eloq /home/eloq/.ssh + install -m 600 -o eloq -g eloq '${CONTROL_REPO_ROOT}/tests/docker_ha/id_ed25519' '${ELOQCTL_CONTROL_SSH_KEY}' + install -d -m 755 -o eloq -g eloq '${CONTROL_ELOQCTL_HOME}' + install -d -m 755 -o eloq -g eloq '${CONTROL_ELOQCTL_HOME}/bin' + cp '${CONTROL_REPO_ROOT}/target/debug/cluster_mgr' '${CONTROL_ELOQCTL}' + chmod 755 '${CONTROL_ELOQCTL}' + rm -f /usr/local/bin/eloqctl + printf '%s\n' '#!/bin/bash' \ + 'export HOME=/home/eloq' \ + 'export ELOQCTL_HOME=/home/eloq/.eloqctl' \ + 'exec /home/eloq/.eloqctl/bin/cluster_mgr "\$@"' \ + > /usr/local/bin/eloqctl + chmod 755 /usr/local/bin/eloqctl + rm -f '${CONTROL_ELOQCTL_HOME}/config' + ln -s '${CONTROL_REPO_ROOT}/src/cluster_mgr/config' '${CONTROL_ELOQCTL_HOME}/config' + cat > /etc/profile.d/eloqctl.sh <<'EOF' +export ELOQCTL_HOME='${CONTROL_ELOQCTL_HOME}' +EOF + grep -qxF 'export ELOQCTL_HOME=${ELOQCTL_HOME:-/home/eloq/.eloqctl}' /home/eloq/.bashrc || \ + printf '\nexport ELOQCTL_HOME=${ELOQCTL_HOME:-/home/eloq/.eloqctl}\n' >> /home/eloq/.bashrc + chown -h eloq:eloq '${CONTROL_ELOQCTL_HOME}/config' + chown -R eloq:eloq '${CONTROL_ELOQCTL_HOME}' + " } start_docker_env() { @@ -69,8 +131,28 @@ start_docker_env() { echo "[docker] Start Docker HA network" compose up -d >/dev/null + echo "[docker] Wait for MinIO" + for _ in $(seq 1 60); do + if curl -fsS "${MINIO_ENDPOINT}/minio/health/live" >/dev/null 2>&1; then + break + fi + sleep 1 + done + curl -fsS "${MINIO_ENDPOINT}/minio/health/live" >/dev/null 2>&1 || { + echo "FAIL: MinIO is not ready at ${MINIO_ENDPOINT}" + compose ps || true + compose logs --no-color --tail=80 minio || true + exit 1 + } + + echo "[docker] Ensure MinIO bucket ${MINIO_BUCKET}" + docker run --rm --network host minio/mc:RELEASE.2025-05-21T01-59-54Z \ + alias set "${MINIO_ALIAS}" "${MINIO_ENDPOINT}" "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" >/dev/null + docker run --rm --network host minio/mc:RELEASE.2025-05-21T01-59-54Z \ + mb --ignore-existing "${MINIO_ALIAS}/${MINIO_BUCKET}" >/dev/null + echo "[docker] Wait for SSH" - for host in 2221 2222 2223; do + for host in 2221 2222 2223 2224; do for _ in $(seq 1 60); do if ssh_cmd "${host}" true >/dev/null 2>&1; then break @@ -84,6 +166,9 @@ start_docker_env() { exit 1 } done + + echo "[docker] Prepare control node ${CONTROL_NODE_SERVICE}" + prepare_control_node } dump_failure_diagnostics() { @@ -117,17 +202,29 @@ run_with_progress() { timeout --kill-after=10s "${timeout_seconds}s" "$@" > "${log_file}" 2>&1 & local cmd_pid=$! local elapsed=0 + local last_cmd_lines=0 + local last_eloq_lines=0 while kill -0 "${cmd_pid}" >/dev/null 2>&1; do sleep 5 elapsed=$((elapsed + 5)) echo " ... still running after ${elapsed}s: $*" if [ -s "${log_file}" ]; then - echo " ---- recent command output ----" - tail -20 "${log_file}" || true + local cmd_lines + cmd_lines=$(wc -l < "${log_file}" 2>/dev/null || echo 0) + if [ "${cmd_lines}" -gt "${last_cmd_lines}" ]; then + echo " ---- new command output ----" + sed -n "$((last_cmd_lines + 1)),$((cmd_lines))p" "${log_file}" || true + last_cmd_lines="${cmd_lines}" + fi fi if [ -f "${ELOQCTL_HOME}/logs/last-launch.log" ]; then - echo " ---- recent eloqctl log ----" - tail -20 "${ELOQCTL_HOME}/logs/last-launch.log" || true + local eloq_lines + eloq_lines=$(wc -l < "${ELOQCTL_HOME}/logs/last-launch.log" 2>/dev/null || echo 0) + if [ "${eloq_lines}" -gt "${last_eloq_lines}" ]; then + echo " ---- new eloqctl log ----" + sed -n "$((last_eloq_lines + 1)),$((eloq_lines))p" "${ELOQCTL_HOME}/logs/last-launch.log" || true + last_eloq_lines="${eloq_lines}" + fi fi done wait "${cmd_pid}" diff --git a/tests/docker_ha/Dockerfile b/tests/docker_ha/Dockerfile index 2f549811..f15a3535 100644 --- a/tests/docker_ha/Dockerfile +++ b/tests/docker_ha/Dockerfile @@ -2,8 +2,21 @@ FROM ubuntu:24.04 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + gdb \ + gdbserver \ + iproute2 \ + less \ + lsof \ + net-tools \ openssh-server \ + procps \ + redis-tools \ + rsync \ + file \ + strace \ sudo \ + vim-tiny \ && rm -rf /var/lib/apt/lists/* RUN useradd -m -s /bin/bash eloq \ @@ -21,6 +34,7 @@ RUN chmod 600 /home/eloq/.ssh/authorized_keys \ && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config \ && printf '\nAuthorizedKeysFile .ssh/authorized_keys\n' >> /etc/ssh/sshd_config \ && printf '* soft nofile 524288\n* hard nofile 524288\n* hard core unlimited\n* soft core unlimited\n' >> /etc/security/limits.conf \ + && ln -sf /etc/ssl/certs /usr/lib/ssl/certs \ && mkdir -p /data \ && chown -R eloq:eloq /data diff --git a/tests/docker_ha/Dockerfile.stress b/tests/docker_ha/Dockerfile.stress new file mode 100644 index 00000000..79c6dfaa --- /dev/null +++ b/tests/docker_ha/Dockerfile.stress @@ -0,0 +1,21 @@ +FROM python:3.13-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gdb \ + gdbserver \ + iproute2 \ + less \ + lsof \ + procps \ + strace \ + && rm -rf /var/lib/apt/lists/* + +COPY tests/e2e/cmd_stress_py/requirements.txt /tmp/cmd_stress_py_requirements.txt + +RUN pip install --no-cache-dir -r /tmp/cmd_stress_py_requirements.txt + +WORKDIR /workspace/eloq_waiter + +CMD ["sleep", "infinity"] diff --git a/tests/docker_ha/Dockerfile.stress_go b/tests/docker_ha/Dockerfile.stress_go new file mode 100644 index 00000000..e022ec5e --- /dev/null +++ b/tests/docker_ha/Dockerfile.stress_go @@ -0,0 +1,16 @@ +FROM golang:1.24-bookworm + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gdb \ + gdbserver \ + iproute2 \ + less \ + lsof \ + procps \ + strace \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace/eloq_waiter +CMD ["sleep", "infinity"] diff --git a/tests/docker_ha/Dockerfile.stress_ts b/tests/docker_ha/Dockerfile.stress_ts new file mode 100644 index 00000000..12ff27e7 --- /dev/null +++ b/tests/docker_ha/Dockerfile.stress_ts @@ -0,0 +1,16 @@ +FROM node:22-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gdb \ + gdbserver \ + iproute2 \ + less \ + lsof \ + procps \ + strace \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace/eloq_waiter +CMD ["sleep", "infinity"] diff --git a/tests/docker_ha/docker-compose.yaml b/tests/docker_ha/docker-compose.yaml index 3beaf56c..d5dd7f93 100644 --- a/tests/docker_ha/docker-compose.yaml +++ b/tests/docker_ha/docker-compose.yaml @@ -1,4 +1,19 @@ services: + minio: + image: minio/minio:RELEASE.2025-02-28T09-55-16Z + container_name: eloqctl-e2e-minio + hostname: minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + command: server /data --console-address ":9001" + ports: + - "127.0.0.1:19000:9000" + - "127.0.0.1:19001:9001" + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.20 + eloq-node-1: build: context: . @@ -13,6 +28,8 @@ services: networks: eloqctl-e2e-net: ipv4_address: 172.28.10.11 + depends_on: + - minio eloq-node-2: image: eloqctl-e2e-ubuntu:24.04 @@ -24,6 +41,7 @@ services: - "127.0.0.1:26372:16380" depends_on: - eloq-node-1 + - minio networks: eloqctl-e2e-net: ipv4_address: 172.28.10.12 @@ -38,10 +56,139 @@ services: - "127.0.0.1:26373:16380" depends_on: - eloq-node-1 + - minio networks: eloqctl-e2e-net: ipv4_address: 172.28.10.13 + eloq-node-4: + image: eloqctl-e2e-ubuntu:24.04 + container_name: eloqctl-e2e-node-4 + hostname: eloq-node-4 + ports: + - "127.0.0.1:2224:22" + - "127.0.0.1:19010:9000" + working_dir: /workspace/eloq_waiter + volumes: + - "../..:/workspace/eloq_waiter" + depends_on: + - eloq-node-1 + - minio + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.14 + + stress-client: + build: + context: ../.. + dockerfile: tests/docker_ha/Dockerfile.stress + image: eloqctl-e2e-stress:latest + container_name: eloqctl-e2e-stress + hostname: stress-client + working_dir: /workspace/eloq_waiter + command: ["sleep", "infinity"] + volumes: + - "../..:/workspace/eloq_waiter" + depends_on: + - eloq-node-1 + - eloq-node-2 + - eloq-node-3 + - eloq-node-4 + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.30 + + stress-client-2: + image: eloqctl-e2e-stress:latest + container_name: eloqctl-e2e-stress-2 + hostname: stress-client-2 + working_dir: /workspace/eloq_waiter + command: ["sleep", "infinity"] + volumes: + - "../..:/workspace/eloq_waiter" + depends_on: + - eloq-node-1 + - eloq-node-2 + - eloq-node-3 + - eloq-node-4 + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.31 + + stress-client-3: + image: eloqctl-e2e-stress:latest + container_name: eloqctl-e2e-stress-3 + hostname: stress-client-3 + working_dir: /workspace/eloq_waiter + command: ["sleep", "infinity"] + volumes: + - "../..:/workspace/eloq_waiter" + depends_on: + - eloq-node-1 + - eloq-node-2 + - eloq-node-3 + - eloq-node-4 + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.32 + + stress-client-4: + image: eloqctl-e2e-stress:latest + container_name: eloqctl-e2e-stress-4 + hostname: stress-client-4 + working_dir: /workspace/eloq_waiter + command: ["sleep", "infinity"] + volumes: + - "../..:/workspace/eloq_waiter" + depends_on: + - eloq-node-1 + - eloq-node-2 + - eloq-node-3 + - eloq-node-4 + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.33 + + stress-go: + build: + context: ../.. + dockerfile: tests/docker_ha/Dockerfile.stress_go + image: eloqctl-e2e-stress-go:latest + container_name: eloqctl-e2e-stress-go + hostname: stress-go + working_dir: /workspace/eloq_waiter + command: ["sleep", "infinity"] + volumes: + - "../..:/workspace/eloq_waiter" + depends_on: + - eloq-node-1 + - eloq-node-2 + - eloq-node-3 + - eloq-node-4 + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.34 + + stress-ts: + build: + context: ../.. + dockerfile: tests/docker_ha/Dockerfile.stress_ts + image: eloqctl-e2e-stress-ts:latest + container_name: eloqctl-e2e-stress-ts + hostname: stress-ts + working_dir: /workspace/eloq_waiter + command: ["sleep", "infinity"] + volumes: + - "../..:/workspace/eloq_waiter" + depends_on: + - eloq-node-1 + - eloq-node-2 + - eloq-node-3 + - eloq-node-4 + networks: + eloqctl-e2e-net: + ipv4_address: 172.28.10.35 + networks: eloqctl-e2e-net: name: eloqctl-e2e-net diff --git a/tests/docker_ha/topology.yaml b/tests/docker_ha/topology.yaml deleted file mode 100644 index 82a5c47d..00000000 --- a/tests/docker_ha/topology.yaml +++ /dev/null @@ -1,61 +0,0 @@ -connection: - username: "eloq" - auth_type: "keypair" - auth: - keypair: "${ELOQCTL_DOCKER_SSH_KEY}" - ssh_endpoints: - 172.28.10.11: - host: "127.0.0.1" - port: 2221 - 172.28.10.12: - host: "127.0.0.1" - port: 2222 - 172.28.10.13: - host: "127.0.0.1" - port: 2223 - service_endpoints: - 172.28.10.11:6379: - host: "127.0.0.1" - port: 16371 - 172.28.10.11:16380: - host: "127.0.0.1" - port: 26371 - 172.28.10.12:6379: - host: "127.0.0.1" - port: 16372 - 172.28.10.12:16380: - host: "127.0.0.1" - port: 26372 - 172.28.10.13:6379: - host: "127.0.0.1" - port: 16373 - 172.28.10.13:16380: - host: "127.0.0.1" - port: 26373 -deployment: - cluster_name: "test-docker-ha" - product: "EloqKV" - version: "1.2.2" - install_dir: "/home/eloq" - enable_wal: false - enable_io_uring: false - cluster_mode: true - checkpoint_interval: 120 - hardware: - 172.28.10.11: - cpu: 2 - memory: 1024 - 172.28.10.12: - cpu: 2 - memory: 1024 - 172.28.10.13: - cpu: 2 - memory: 1024 - tx_service: - tx_host_ports: [172.28.10.11:6379] - standby_host_ports: [172.28.10.12:6379] - voter_host_ports: [172.28.10.13:6379] - requirepass: "testpass" - storage_service: - rocksdb: !LOCAL - path: "/home/eloq/rocksdb" diff --git a/tests/e2e/cmd_stress_go/go.mod b/tests/e2e/cmd_stress_go/go.mod new file mode 100644 index 00000000..d0b32f97 --- /dev/null +++ b/tests/e2e/cmd_stress_go/go.mod @@ -0,0 +1,10 @@ +module eloq_waiter/tests/e2e/cmd_stress_go + +go 1.24 + +require github.com/redis/go-redis/v9 v9.7.3 + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect +) diff --git a/tests/e2e/cmd_stress_go/go.sum b/tests/e2e/cmd_stress_go/go.sum new file mode 100644 index 00000000..bc52aa29 --- /dev/null +++ b/tests/e2e/cmd_stress_go/go.sum @@ -0,0 +1,10 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= diff --git a/tests/e2e/cmd_stress_go/main.go b/tests/e2e/cmd_stress_go/main.go new file mode 100644 index 00000000..408a3c16 --- /dev/null +++ b/tests/e2e/cmd_stress_go/main.go @@ -0,0 +1,882 @@ +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "log" + "os" + "os/signal" + "sort" + "strings" + "sync" + "syscall" + "time" + + "github.com/redis/go-redis/v9" +) + +// cmdTestCase defines a single command test. +type cmdTestCase struct { + Name string + Fn func(ctx context.Context, c redis.UniversalClient, keyIdx int) error +} + +// --------------------------------------------------------------------------- +// Command definitions covering eloqkv's supported Redis commands +// --------------------------------------------------------------------------- +func k(i int) string { return fmt.Sprintf("stress:cmd:%d", i) } +func kt(i int, suffix string) string { return fmt.Sprintf("{stress:cmd:%d}%s", i, suffix) } +func mk(i int) map[string]interface{} { + return map[string]interface{}{"f0": "v0", "f1": "v1", "f2": "v2", "f3": "v3"} +} + +func cmdTests(replicaAddr string) []cmdTestCase { + return []cmdTestCase{ + // ── Connection / Server ── + {"PING", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Ping(ctx).Err() + }}, + {"ECHO", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.Echo(ctx, fmt.Sprintf("%d", i)).Result() + return err + }}, + {"DBSIZE", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.DBSize(ctx).Result() + return err + }}, + {"TIME", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.Time(ctx).Result() + return err + }}, + {"INFO", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Info(ctx).Err() + }}, + {"CLUSTER_INFO", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Do(ctx, "CLUSTER", "INFO").Err() + }}, + + // ── String ── + {"SET", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Set(ctx, k(i), i, 0).Err() + }}, + {"GET", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Get(ctx, k(i)).Err() + }}, + {"GETDEL", func(ctx context.Context, c redis.UniversalClient, i int) error { + key := k(i) + "gd" + c.Set(ctx, key, "x", 0) + return c.GetDel(ctx, key).Err() + }}, + {"SETNX", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SetNX(ctx, k(i)+"nx", i, 0).Result() + return err + }}, + {"SETEX", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SetEx(ctx, k(i)+"ex", i, 60*time.Second).Err() + }}, + {"PSETEX", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Do(ctx, "PSETEX", k(i)+"px", int64(60000), fmt.Sprintf("%d", i)).Err() + }}, + {"STRLEN", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.StrLen(ctx, k(i)).Result() + return err + }}, + {"GETRANGE", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.GetRange(ctx, k(i), 0, 3).Result() + return err + }}, + {"SETRANGE", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SetRange(ctx, k(i)+"sr", 0, "xx").Result() + return err + }}, + {"APPEND", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.Append(ctx, k(i)+"ap", "tail").Result() + return err + }}, + {"GETBIT", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.GetBit(ctx, k(i), 0).Result() + return err + }}, + {"SETBIT", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SetBit(ctx, k(i)+"bi", 7, 1).Result() + return err + }}, + {"BITCOUNT", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.BitCount(ctx, k(i), nil).Result() + return err + }}, + {"BITPOS", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.BitPos(ctx, k(i), 1).Result() + return err + }}, + {"INCR", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.Incr(ctx, k(i)+"ctr").Result() + return err + }}, + {"DECR", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.Decr(ctx, k(i)+"ctr").Result() + return err + }}, + {"INCRBY", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.IncrBy(ctx, k(i)+"ctr", 10).Result() + return err + }}, + {"DECRBY", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.DecrBy(ctx, k(i)+"ctr", 5).Result() + return err + }}, + {"INCRBYFLOAT", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.IncrByFloat(ctx, k(i)+"fl", 0.5).Result() + return err + }}, + {"MSET", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.MSet(ctx, kt(i, "m0"), "a", kt(i, "m1"), "b").Err() + }}, + {"MGET", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.MGet(ctx, kt(i, "m0"), kt(i, "m1")).Err() + }}, + + // ── Hash ── + {"HSET", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.HSet(ctx, k(i)+"h", mk(i)).Result() + return err + }}, + {"HGET", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.HGet(ctx, k(i)+"h", "f0").Err() + }}, + {"HDEL", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.HSet(ctx, k(i)+"hd", "f0", "x") + _, err := c.HDel(ctx, k(i)+"hd", "f0").Result() + return err + }}, + {"HEXISTS", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.HExists(ctx, k(i)+"h", "f0").Result() + return err + }}, + {"HGETALL", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.HGetAll(ctx, k(i)+"h").Err() + }}, + {"HLEN", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.HLen(ctx, k(i)+"h").Result() + return err + }}, + {"HSTRLEN", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Do(ctx, "HSTRLEN", k(i)+"h", "f0").Err() + }}, + {"HINCRBY", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.HSet(ctx, k(i)+"hc", "cnt", 0) + _, err := c.HIncrBy(ctx, k(i)+"hc", "cnt", 1).Result() + return err + }}, + {"HINCRBYFLOAT", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.HSet(ctx, k(i)+"hf", "val", "0") + _, err := c.HIncrByFloat(ctx, k(i)+"hf", "val", 0.5).Result() + return err + }}, + {"HMGET", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.HMGet(ctx, k(i)+"h", "f0", "f1").Err() + }}, + {"HKEYS", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.HKeys(ctx, k(i)+"h").Err() + }}, + {"HVALS", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.HVals(ctx, k(i)+"h").Err() + }}, + {"HSETNX", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.HSetNX(ctx, k(i)+"hx", "uniq", i).Result() + return err + }}, + + // ── List ── + {"LPUSH", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.LPush(ctx, k(i)+"l", i, i+1).Result() + return err + }}, + {"RPUSH", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.RPush(ctx, k(i)+"r", i, i+1).Result() + return err + }}, + {"LPOP", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.LPop(ctx, k(i)+"l").Err() + }}, + {"RPOP", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.RPop(ctx, k(i)+"r").Err() + }}, + {"LLEN", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.LLen(ctx, k(i)+"l").Result() + return err + }}, + {"LRANGE", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.LRange(ctx, k(i)+"l", 0, -1).Err() + }}, + {"LINDEX", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.LIndex(ctx, k(i)+"l", 0).Err() + }}, + {"LSET", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.LPush(ctx, k(i)+"ls", "x") + return c.LSet(ctx, k(i)+"ls", 0, "UPD").Err() + }}, + {"LTRIM", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.LTrim(ctx, k(i)+"lt", 0, 1).Err() + }}, + {"LREM", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.LRem(ctx, k(i)+"l", 1, fmt.Sprintf("%d", i)).Result() + return err + }}, + {"LPUSHX", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.LPushX(ctx, k(i)+"l", "xx").Result() + return err + }}, + {"RPUSHX", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.RPushX(ctx, k(i)+"r", "xx").Result() + return err + }}, + {"RPOPLPUSH", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.RPush(ctx, kt(i, "r"), "x") + return c.RPopLPush(ctx, kt(i, "r"), kt(i, "rl")).Err() + }}, + + // ── Set ── + {"SADD", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SAdd(ctx, k(i)+"s", i, i+1).Result() + return err + }}, + {"SMEMBERS", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SMembers(ctx, k(i)+"s").Err() + }}, + {"SCARD", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SCard(ctx, k(i)+"s").Result() + return err + }}, + {"SISMEMBER", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SIsMember(ctx, k(i)+"s", i).Result() + return err + }}, + {"SREM", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SRem(ctx, k(i)+"s", fmt.Sprintf("%d", i)).Result() + return err + }}, + {"SPOP", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SPop(ctx, k(i)+"s").Err() + }}, + {"SRANDMEMBER", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.SAdd(ctx, kt(i, "s"), "x") + return c.SRandMember(ctx, kt(i, "s")).Err() + }}, + {"SMOVE", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SMove(ctx, kt(i, "s"), kt(i, "s2"), "shared").Result() + return err + }}, + {"SUNION", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SUnion(ctx, kt(i, "s"), kt(i, "s2")).Err() + }}, + {"SUNIONSTORE", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SUnionStore(ctx, kt(i, "su"), kt(i, "s"), kt(i, "s2")).Result() + return err + }}, + {"SINTER", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SInter(ctx, kt(i, "s"), kt(i, "s2")).Err() + }}, + {"SDIFF", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SDiff(ctx, kt(i, "s"), kt(i, "s2")).Err() + }}, + {"SMISMEMBER", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SMIsMember(ctx, kt(i, "s"), fmt.Sprintf("%d", i), fmt.Sprintf("%d", i+1)).Err() + }}, + {"SINTERCARD", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.SInterCard(ctx, 2, kt(i, "s"), kt(i, "s2")).Result() + return err + }}, + {"SSCAN", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.SScan(ctx, k(i)+"s", 0, "", 10).Err() + }}, + + // ── Sorted Set ── + {"ZADD", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZAdd(ctx, k(i)+"z", redis.Z{Score: float64(i), Member: fmt.Sprintf("%d", i)}).Result() + return err + }}, + {"ZCARD", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZCard(ctx, k(i)+"z").Result() + return err + }}, + {"ZCOUNT", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZCount(ctx, k(i)+"z", "0", "100").Result() + return err + }}, + {"ZSCORE", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.ZAdd(ctx, k(i)+"z", redis.Z{Score: float64(i % 50), Member: fmt.Sprintf("%d", i%50)}) + return c.ZScore(ctx, k(i)+"z", fmt.Sprintf("%d", i%50)).Err() + }}, + {"ZRANK", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.ZAdd(ctx, k(i)+"z", redis.Z{Score: float64(i % 50), Member: fmt.Sprintf("%d", i%50)}) + _, err := c.ZRank(ctx, k(i)+"z", fmt.Sprintf("%d", i%50)).Result() + return err + }}, + {"ZREVRANK", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.ZAdd(ctx, k(i)+"z", redis.Z{Score: float64(i % 50), Member: fmt.Sprintf("%d", i%50)}) + _, err := c.ZRevRank(ctx, k(i)+"z", fmt.Sprintf("%d", i%50)).Result() + return err + }}, + {"ZRANGE", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.ZRange(ctx, k(i)+"z", 0, -1).Err() + }}, + {"ZRANGEBYSCORE", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.ZRangeByScore(ctx, k(i)+"z", &redis.ZRangeBy{Min: "0", Max: "100"}).Err() + }}, + {"ZREVRANGE", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.ZRevRange(ctx, k(i)+"z", 0, -1).Err() + }}, + {"ZREVRANGEBYSCORE", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.ZRevRangeByScore(ctx, k(i)+"z", &redis.ZRangeBy{Min: "0", Max: "100"}).Err() + }}, + {"ZINCRBY", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZIncrBy(ctx, k(i)+"z", 1, fmt.Sprintf("%d", i%50)).Result() + return err + }}, + {"ZREM", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZRem(ctx, k(i)+"zr", fmt.Sprintf("%d", i%5)).Result() + return err + }}, + {"ZREMRANGEBYSCORE", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZRemRangeByScore(ctx, k(i)+"zr", "0", "50").Result() + return err + }}, + {"ZREMRANGEBYRANK", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZRemRangeByRank(ctx, k(i)+"zr", 0, 1).Result() + return err + }}, + {"ZPOPMIN", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.ZPopMin(ctx, k(i)+"z", 1).Err() + }}, + {"ZRANDMEMBER", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.ZRandMember(ctx, k(i)+"z", 1).Err() + }}, + {"ZMSCORE", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.ZAdd(ctx, k(i)+"z", redis.Z{Score: float64(i % 50), Member: fmt.Sprintf("%d", i%50)}, + redis.Z{Score: float64(i%50 + 1), Member: fmt.Sprintf("%d", i%50+1)}) + return c.ZMScore(ctx, k(i)+"z", fmt.Sprintf("%d", i%50), fmt.Sprintf("%d", i%50+1)).Err() + }}, + {"ZLEXCOUNT", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.ZLexCount(ctx, k(i)+"zl", "-", "+").Result() + return err + }}, + {"ZSCAN", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.ZScan(ctx, k(i)+"z", 0, "", 10).Err() + }}, + + // ── Generic / Key ── + {"DEL", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.Set(ctx, k(i)+"td", "x", 0) + _, err := c.Del(ctx, k(i)+"td").Result() + return err + }}, + {"UNLINK", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.Set(ctx, k(i)+"tu", "x", 0) + _, err := c.Unlink(ctx, k(i)+"tu").Result() + return err + }}, + {"EXISTS", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.Exists(ctx, k(i)).Result() + return err + }}, + {"TYPE", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.Type(ctx, k(i)).Result() + return err + }}, + {"EXPIRE", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.Set(ctx, k(i)+"te", "x", 0) + _, err := c.Expire(ctx, k(i)+"te", 300*time.Second).Result() + return err + }}, + {"PEXPIRE", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.Set(ctx, k(i)+"tp", "x", 0) + _, err := c.PExpire(ctx, k(i)+"tp", 300000*time.Millisecond).Result() + return err + }}, + {"EXPIREAT", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.Set(ctx, k(i)+"tea", "x", 0) + _, err := c.ExpireAt(ctx, k(i)+"tea", time.Now().Add(300*time.Second)).Result() + return err + }}, + {"PEXPIREAT", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.Set(ctx, k(i)+"tpa", "x", 0) + _, err := c.PExpireAt(ctx, k(i)+"tpa", time.Now().Add(300000*time.Millisecond)).Result() + return err + }}, + {"TTL", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.TTL(ctx, k(i)).Result() + return err + }}, + {"PTTL", func(ctx context.Context, c redis.UniversalClient, i int) error { + _, err := c.PTTL(ctx, k(i)).Result() + return err + }}, + {"PERSIST", func(ctx context.Context, c redis.UniversalClient, i int) error { + key := k(i) + "tpe" + c.Set(ctx, key, "x", 0) + c.Expire(ctx, key, 300*time.Second) + _, err := c.Persist(ctx, key).Result() + return err + }}, + {"KEYS", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Keys(ctx, k(i)+"*").Err() + }}, + {"SCAN", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Scan(ctx, 0, "stress:cmd:*", 10).Err() + }}, + {"SORT", func(ctx context.Context, c redis.UniversalClient, i int) error { + c.LPush(ctx, k(i)+"so", "2", "1", "3") + return c.Sort(ctx, k(i)+"so", &redis.Sort{Alpha: true}).Err() + }}, + {"DUMP", func(ctx context.Context, c redis.UniversalClient, i int) error { + return c.Dump(ctx, k(i)).Err() + }}, + {"RESTORE", func(ctx context.Context, c redis.UniversalClient, i int) error { + payload, err := c.Dump(ctx, k(i)).Bytes() + if err != nil { + return err + } + return c.RestoreReplace(ctx, kt(i, "rs"), 0, string(payload)).Err() + }}, + } +} + +func selectCmdTests(replicaAddr string, commandSet string) []cmdTestCase { + tests := cmdTests(replicaAddr) + if commandSet == "info-only" { + return []cmdTestCase{tests[4]} + } + return tests +} + +// --------------------------------------------------------------------------- +// Stats +// --------------------------------------------------------------------------- +type cmdStats struct { + mu sync.Mutex + counts map[string]*cmdStatEntry +} + +type cmdStatEntry struct { + ok int64 + fail int64 + errorTypes map[string]int64 + samples []string +} + +type errorCount struct { + signature string + count int64 +} + +const errorSampleLimit = 3 +const errorTypeLimit = 5 + +func newCmdStats() *cmdStats { + return &cmdStats{counts: map[string]*cmdStatEntry{}} +} + +func (s *cmdStats) addOK(name string) { + s.mu.Lock() + c := s.getOrCreateLocked(name) + c.ok++ + s.mu.Unlock() +} + +func (s *cmdStats) addFail(name string, err error) { + s.mu.Lock() + c := s.getOrCreateLocked(name) + c.fail++ + signature := errorSignature(err) + c.errorTypes[signature]++ + if len(c.samples) < errorSampleLimit && !contains(c.samples, signature) { + c.samples = append(c.samples, signature) + } + s.mu.Unlock() +} + +func (s *cmdStats) getOrCreateLocked(name string) *cmdStatEntry { + if c, ok := s.counts[name]; ok { + return c + } + c := &cmdStatEntry{errorTypes: map[string]int64{}} + s.counts[name] = c + return c +} + +func (s *cmdStats) snapshot() map[string]cmdStatEntry { + s.mu.Lock() + defer s.mu.Unlock() + out := map[string]cmdStatEntry{} + for k, v := range s.counts { + errorTypes := map[string]int64{} + for sig, count := range v.errorTypes { + errorTypes[sig] = count + } + samples := append([]string(nil), v.samples...) + out[k] = cmdStatEntry{ok: v.ok, fail: v.fail, errorTypes: errorTypes, samples: samples} + } + return out +} + +func errorSignature(err error) string { + if err == nil { + return "" + } + msg := strings.Join(strings.Fields(err.Error()), " ") + if msg == "" { + msg = fmt.Sprintf("%v", err) + } + return fmt.Sprintf("%T: %s", err, msg) +} + +func contains(items []string, target string) bool { + for _, item := range items { + if item == target { + return true + } + } + return false +} + +func topErrorCounts(errorTypes map[string]int64) []errorCount { + counts := make([]errorCount, 0, len(errorTypes)) + for signature, count := range errorTypes { + counts = append(counts, errorCount{signature: signature, count: count}) + } + sort.Slice(counts, func(i, j int) bool { + if counts[i].count == counts[j].count { + return counts[i].signature < counts[j].signature + } + return counts[i].count > counts[j].count + }) + if len(counts) > errorTypeLimit { + counts = counts[:errorTypeLimit] + } + return counts +} + +// --------------------------------------------------------------------------- +// Worker +// --------------------------------------------------------------------------- +func stressWorker(ctx context.Context, client redis.UniversalClient, tests []cmdTestCase, + stats *cmdStats, wg *sync.WaitGroup, startKeyIdx, keyMod, repeat int) { + defer wg.Done() + cmdIdx := 0 + keyIdx := startKeyIdx + for { + select { + case <-ctx.Done(): + return + default: + } + tc := tests[cmdIdx%len(tests)] + ki := keyIdx % keyMod + for r := 0; r < repeat; r++ { + select { + case <-ctx.Done(): + return + default: + } + err := tc.Fn(ctx, client, ki) + if err != nil { + stats.addFail(tc.Name, err) + } else { + stats.addOK(tc.Name) + } + } + cmdIdx++ + if cmdIdx%len(tests) == 0 { + keyIdx++ + } + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +func main() { + var startupNodes string + var password string + var workers int + var duration time.Duration + var progressInterval time.Duration + var keyCount int + var insecureTLS bool + var cmdTimeout time.Duration + var inflight int + var repeat int + var logPrefix string + var readFromReplicas bool + var commandSet string + + flag.StringVar(&startupNodes, "startup-nodes", "127.0.0.1:6379,127.0.0.1:6380", + "comma-separated startup nodes (host:port)") + flag.StringVar(&password, "password", "testpass", "redis password") + flag.IntVar(&workers, "workers", 16, "number of concurrent stress workers") + flag.IntVar(&inflight, "inflight", 4, "concurrent execution slots per worker") + flag.DurationVar(&duration, "duration", 300*time.Second, "test duration") + flag.DurationVar(&progressInterval, "progress-interval", 5*time.Second, "progress print interval") + flag.IntVar(&keyCount, "key-count", 256, "key space size") + flag.BoolVar(&insecureTLS, "tls-insecure", true, "skip TLS verification") + flag.DurationVar(&cmdTimeout, "cmd-timeout", 5*time.Second, "per-command timeout") + flag.IntVar(&repeat, "repeat", 10, "times to repeat each command per round") + flag.StringVar(&logPrefix, "log-prefix", "", "optional prefix in log lines") + flag.BoolVar(&readFromReplicas, "read-from-replicas", false, "route read-only standalone requests to a discovered replica when available") + flag.StringVar(&commandSet, "command-set", "full", "command mix to run: full or info-only") + flag.Parse() + + logger := log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + addrs := strings.Split(startupNodes, ",") + for i := range addrs { + addrs[i] = strings.TrimSpace(addrs[i]) + } + if len(addrs) < 1 { + logger.Fatalf("need at least 1 startup node, got %q", startupNodes) + } + + logger.Printf("%sstarting eloqkv command stress test", pref(logPrefix)) + logger.Printf("%snodes=%s workers=%d inflight=%d duration=%s key_count=%d command_set=%s read_from_replicas=%t", + pref(logPrefix), strings.Join(addrs, ","), workers, inflight, duration, keyCount, commandSet, readFromReplicas) + + totalSlots := workers * inflight + + // Cluster client for discovery + clusterOpts := &redis.ClusterOptions{ + Addrs: addrs, + Password: password, + DialTimeout: cmdTimeout, + ReadTimeout: cmdTimeout, + WriteTimeout: cmdTimeout, + PoolSize: totalSlots * 2, + MinIdleConns: totalSlots, + ReadOnly: true, + } + if insecureTLS { + clusterOpts.TLSConfig = &tls.Config{InsecureSkipVerify: true} + } + clusterClient := redis.NewClusterClient(clusterOpts) + defer clusterClient.Close() + + if err := clusterClient.Ping(ctx).Err(); err != nil { + logger.Fatalf("cluster ping failed: %v", err) + } + logger.Printf("%scluster ping OK", pref(logPrefix)) + + // Discover master/replica + nodes, err := clusterClient.ClusterNodes(ctx).Result() + if err != nil { + logger.Fatalf("cluster nodes failed: %v", err) + } + var masterAddr, replicaAddr string + for _, line := range strings.Split(nodes, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + addr := strings.Split(fields[1], "@")[0] + if strings.Contains(fields[2], "master") && masterAddr == "" { + masterAddr = addr + } + if strings.Contains(fields[2], "slave") && replicaAddr == "" { + replicaAddr = addr + } + } + if masterAddr == "" { + logger.Fatalf("no master found in cluster nodes") + } + + logger.Printf("%smaster=%s replica=%s", pref(logPrefix), masterAddr, replicaAddr) + + clientOpts := func() *redis.Options { + return &redis.Options{ + Password: password, + DialTimeout: cmdTimeout, + ReadTimeout: cmdTimeout, + WriteTimeout: cmdTimeout, + PoolSize: totalSlots * 2, + MinIdleConns: totalSlots, + } + } + + masterOpts := clientOpts() + masterOpts.Addr = masterAddr + if insecureTLS { + masterOpts.TLSConfig = &tls.Config{InsecureSkipVerify: true} + } + masterClient := redis.NewClient(masterOpts) + defer masterClient.Close() + + var replicaClient *redis.Client + if replicaAddr != "" && replicaAddr != masterAddr { + replicaOpts := clientOpts() + replicaOpts.Addr = replicaAddr + if insecureTLS { + replicaOpts.TLSConfig = &tls.Config{InsecureSkipVerify: true} + } + replicaClient = redis.NewClient(replicaOpts) + defer replicaClient.Close() + } + + // Preload keys + logger.Printf("%sPreloading %d keys ...", pref(logPrefix), keyCount) + for i := 0; i < keyCount; i++ { + if err := masterClient.Set(ctx, k(i), i, 0).Err(); err != nil { + logger.Fatalf("preload key %d failed: %v", i, err) + } + masterClient.HSet(ctx, k(i)+"h", mk(i)) + masterClient.LPush(ctx, k(i)+"l", i) + masterClient.RPush(ctx, k(i)+"r", i) + masterClient.LPush(ctx, k(i)+"lt", "a", "b", "c") + masterClient.SAdd(ctx, k(i)+"s", i) + masterClient.SAdd(ctx, k(i)+"s2", "shared") + masterClient.ZAdd(ctx, k(i)+"z", redis.Z{Score: float64(i % 50), Member: fmt.Sprintf("%d", i%50)}) + for j := i % 5; j < i%5+3; j++ { + masterClient.ZAdd(ctx, k(i)+"zr", redis.Z{Score: float64(j), Member: fmt.Sprintf("%d", j)}) + } + } + logger.Printf("%sKeyspace ready", pref(logPrefix)) + + tests := selectCmdTests(replicaAddr, commandSet) + + // Quick coverage check — standalone + logger.Printf("%sRunning coverage check (standalone) ...", pref(logPrefix)) + standaloneCovOK, standaloneCovFail := 0, 0 + var standaloneFailedCmds []string + for _, tc := range tests { + err := tc.Fn(ctx, masterClient, 0) + if err != nil { + standaloneCovFail++ + standaloneFailedCmds = append(standaloneFailedCmds, fmt.Sprintf("%s(%v)", tc.Name, err)) + } else { + standaloneCovOK++ + } + } + logger.Printf("%s standalone coverage: %d/%d passed", pref(logPrefix), standaloneCovOK, standaloneCovOK+standaloneCovFail) + for _, fc := range standaloneFailedCmds { + logger.Printf("%s FAIL %s", pref(logPrefix), fc) + } + + // Quick coverage check — cluster + logger.Printf("%sRunning coverage check (cluster) ...", pref(logPrefix)) + clusterCovOK, clusterCovFail := 0, 0 + var clusterFailedCmds []string + for _, tc := range tests { + err := tc.Fn(ctx, clusterClient, 0) + if err != nil { + clusterCovFail++ + clusterFailedCmds = append(clusterFailedCmds, fmt.Sprintf("%s(%v)", tc.Name, err)) + } else { + clusterCovOK++ + } + } + logger.Printf("%s cluster coverage: %d/%d passed", pref(logPrefix), clusterCovOK, clusterCovOK+clusterCovFail) + for _, fc := range clusterFailedCmds { + logger.Printf("%s FAIL %s", pref(logPrefix), fc) + } + + // Start stress workers + standaloneStats := newCmdStats() + clusterStats := newCmdStats() + nStandalone := workers / 2 + nCluster := workers - nStandalone + totalStandaloneSlots := nStandalone * inflight + totalClusterSlots := nCluster * inflight + logger.Printf("%sStarting %d standalone + %d cluster workers with inflight=%d (%d + %d execution slots) ...", + pref(logPrefix), nStandalone, nCluster, inflight, totalStandaloneSlots, totalClusterSlots) + testCtx, cancel := context.WithTimeout(ctx, duration) + defer cancel() + + var wg sync.WaitGroup + for w := 0; w < totalStandaloneSlots; w++ { + wg.Add(1) + go stressWorker(testCtx, masterClient, tests, standaloneStats, &wg, w, keyCount, repeat) + } + for w := 0; w < totalClusterSlots; w++ { + wg.Add(1) + go stressWorker(testCtx, clusterClient, tests, clusterStats, &wg, w+totalStandaloneSlots, keyCount, repeat) + } + + // Progress reporter + go func() { + ticker := time.NewTicker(progressInterval) + defer ticker.Stop() + start := time.Now() + for { + select { + case <-testCtx.Done(): + return + case <-ticker.C: + snap := standaloneStats.snapshot() + var sOK, sFail int64 + for _, v := range snap { + sOK += v.ok + sFail += v.fail + } + csnap := clusterStats.snapshot() + var cOK, cFail int64 + for _, v := range csnap { + cOK += v.ok + cFail += v.fail + } + logger.Printf("%sprogress elapsed=%s standalone ok=%d fail=%d cluster ok=%d fail=%d", + pref(logPrefix), + time.Since(start).Truncate(time.Second), + sOK, sFail, cOK, cFail) + } + } + }() + + wg.Wait() + + // Report + printResult := func(label string, s *cmdStats) (int64, int64) { + snap := s.snapshot() + var totalOK, totalFail int64 + for _, v := range snap { + totalOK += v.ok + totalFail += v.fail + } + logger.Printf("--- %s ---", label) + logger.Printf("total_commands: ok=%d fail=%d", totalOK, totalFail) + for name, v := range snap { + if v.fail > 0 { + logger.Printf(" %s: ok=%d fail=%d", name, v.ok, v.fail) + for _, ec := range topErrorCounts(v.errorTypes) { + logger.Printf(" error[%d]: %.200s", ec.count, ec.signature) + } + for _, sample := range v.samples { + logger.Printf(" sample: %.200s", sample) + } + } + } + return totalOK, totalFail + } + sOK, sFail := printResult("Standalone Client Results", standaloneStats) + cOK, cFail := printResult("Cluster Client Results", clusterStats) + totalOK := sOK + cOK + totalFail := sFail + cFail + + // Tolerate <2% sporadic failures (race conditions from concurrent key access) + if totalFail > 0 && (totalOK == 0 || float64(totalFail)/float64(totalOK+totalFail) > 0.02) { + os.Exit(1) + } + logger.Printf("PASS") +} + +func pref(s string) string { + if s == "" { + return "" + } + return "[" + s + "] " +} diff --git a/tests/e2e/cmd_stress_py/main.py b/tests/e2e/cmd_stress_py/main.py new file mode 100755 index 00000000..1085d4be --- /dev/null +++ b/tests/e2e/cmd_stress_py/main.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python3 +"""Full-command stress test for eloqkv cluster covering Python redis-py SDK. + +Runs comprehensive Redis commands across all supported families: + string, hash, list, set, sorted-set, generic/key, TTL, server, pub/sub. +""" + +import argparse +import concurrent.futures +from collections import Counter +import json +import os +import signal +import ssl +import sys +import threading +import time +import traceback +from typing import Any, Callable, Dict, List, Optional, Tuple + +from redis import Redis +from redis.cluster import ClusterNode, RedisCluster + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- +parser = argparse.ArgumentParser(description="eloqkv full-command stress test") +parser.add_argument("--startup-node", action="append", default=[], + help="Cluster startup node in host:port form (repeatable)") +parser.add_argument("--password", default="testpass") +parser.add_argument("--cmd-timeout", type=float, default=5.0) +parser.add_argument("--progress-interval", type=float, default=5.0) +parser.add_argument("--key-count", type=int, default=256) +parser.add_argument("--tls", action="store_true") +parser.add_argument("--read-from-replicas", action="store_true") +parser.add_argument("--command-set", choices=["full", "info-only"], default="full", + help="Command mix to run during coverage and stress phases") +parser.add_argument("--base-connections", type=int, default=20) +parser.add_argument("--target-connections", type=int, default=10000) +parser.add_argument("--ramp-per-second", type=float, default=10.0) +parser.add_argument("--ramp-workers", type=int, default=4) +parser.add_argument("--phase1-qps", type=float, default=5000.0) +parser.add_argument("--phase2-qps", type=float, default=5000.0) +parser.add_argument("--duration", type=int, default=0, + help="Phase-2 seconds; <=0 means run until interrupted") +parser.add_argument("--workers", type=int, default=16, + help="Worker threads for command execution") +parser.add_argument("--inflight", type=int, default=4, + help="Concurrent execution slots per worker") +parser.add_argument("--repeat", type=int, default=10, + help="Times to repeat each command per round (higher = more concurrency per command)") +parser.add_argument("--skip-cmd-coverage", action="store_true", + help="Skip full command coverage (only SET/GET)") +parser.add_argument("--results-file", default="", + help="Write JSON results to file") +args = parser.parse_args() + +if not args.startup_node: + print("FAIL: at least one --startup-node is required", flush=True) + sys.exit(1) + +TLS_KWARGS: Dict[str, Any] = {} +if args.tls: + TLS_KWARGS = {"ssl": True, "ssl_cert_reqs": ssl.CERT_NONE, + "ssl_check_hostname": False} + +def parse_cluster_node(value: str) -> ClusterNode: + host, sep, port_str = value.rpartition(":") + if not sep or not host or not port_str: + raise ValueError(f"invalid startup node {value!r}, expected host:port") + return ClusterNode(host, int(port_str)) + + +startup_nodes = [parse_cluster_node(node) for node in args.startup_node if node] +master_node = startup_nodes[0] +replica_node = startup_nodes[1] if len(startup_nodes) > 1 else startup_nodes[0] + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def build_client(node: ClusterNode, decode: bool = True) -> Redis: + max_connections = max(64, args.workers * args.inflight * 8) + return Redis( + host=node.host, port=node.port, + password=args.password, + socket_timeout=args.cmd_timeout, + socket_connect_timeout=args.cmd_timeout, + max_connections=max_connections, + decode_responses=decode, **TLS_KWARGS, + ) + + +def build_cluster_client() -> RedisCluster: + max_connections = max(64, args.workers * args.inflight * 8) + return RedisCluster( + startup_nodes=startup_nodes, + password=args.password, + socket_timeout=args.cmd_timeout, + socket_connect_timeout=args.cmd_timeout, + max_connections=max_connections, + read_from_replicas=args.read_from_replicas, + decode_responses=True, **TLS_KWARGS, + ) + + +def close_client(c: Redis) -> None: + try: c.close() + except Exception: pass + + +ERROR_SAMPLE_LIMIT = 3 +ERROR_TYPE_LIMIT = 5 + + +def make_cmd_stats() -> Dict[str, Dict[str, Any]]: + return { + name: {"ok": 0, "fail": 0, "error_types": Counter(), "samples": []} + for name in _CMD_ORDER + } + + +def record_failure(stats: Dict[str, Dict[str, Any]], cmd_name: str, err: Exception) -> None: + entry = stats[cmd_name] + entry["fail"] += 1 + err_type = type(err).__name__ + message = " ".join(str(err).split()) or repr(err) + signature = f"{err_type}: {message}" + entry["error_types"][signature] += 1 + samples: List[str] = entry["samples"] + if len(samples) < ERROR_SAMPLE_LIMIT and signature not in samples: + samples.append(signature) + +# --------------------------------------------------------------------------- +# Command coverage definitions +# Each entry: (name, callable(client, index) -> result) +# --------------------------------------------------------------------------- +def _k(i: int) -> str: + return f"stress:cmd:{i}" + +def _kt(i: int, suffix: str = "") -> str: + """Hash-tagged key for multi-key commands to share same slot.""" + return f"{{stress:cmd:{i}}}{suffix}" + +def _mk(i: int) -> Dict[str, str]: + return {f"f{j}": f"v{j}" for j in range(4)} + +Str = str + +COMMAND_TESTS: List[Tuple[str, Callable[[Redis, int], Any]]] = [ + # ── Connection / Server ── + ("PING", lambda c, i: c.ping()), + ("ECHO", lambda c, i: c.echo(Str(i))), + ("DBSIZE", lambda c, i: c.dbsize()), + ("TIME", lambda c, i: c.time()), + ("READONLY", lambda c, i: c.execute_command("READONLY")), + ("INFO", lambda c, i: c.info()), + ("CLUSTER_INFO", lambda c, i: c.cluster_info() if hasattr(c, 'cluster_info') else c.execute_command("CLUSTER", "INFO")), + + # ── String ── + ("SET", lambda c, i: c.set(_k(i), Str(i))), + ("GET", lambda c, i: c.get(_k(i))), + ("GETDEL", lambda c, i: (c.set(_k(i) + "gd", "x"), c.getdel(_k(i) + "gd"))[1]), + ("SETNX", lambda c, i: c.setnx(_k(i) + "nx", Str(i))), + ("SETEX", lambda c, i: c.setex(_k(i) + "ex", 60, Str(i))), + ("PSETEX", lambda c, i: c.psetex(_k(i) + "px", 60000, Str(i))), + ("STRLEN", lambda c, i: c.strlen(_k(i))), + ("GETRANGE",lambda c, i: c.getrange(_k(i), 0, 3)), + ("SETRANGE",lambda c, i: c.setrange(_k(i) + "sr", 0, "xx")), + ("APPEND", lambda c, i: c.append(_k(i) + "ap", "tail")), + ("GETBIT", lambda c, i: c.getbit(_k(i), 0)), + ("SETBIT", lambda c, i: c.setbit(_k(i) + "bi", 7, 1)), + ("BITCOUNT",lambda c, i: c.bitcount(_k(i))), + ("BITPOS", lambda c, i: c.bitpos(_k(i), 1)), + ("INCR", lambda c, i: c.incr(_k(i) + "ctr")), + ("DECR", lambda c, i: c.decr(_k(i) + "ctr")), + ("INCRBY", lambda c, i: c.incrby(_k(i) + "ctr", 10)), + ("DECRBY", lambda c, i: c.decrby(_k(i) + "ctr", 5)), + ("INCRBYFLOAT", lambda c, i: c.incrbyfloat(_k(i) + "fl", 0.5)), + ("MSET", lambda c, i: c.mset({_kt(i, "m0"): "a", _kt(i, "m1"): "b"})), + ("MGET", lambda c, i: c.mget([_kt(i, "m0"), _kt(i, "m1")])), + + # ── Hash ── + ("HSET", lambda c, i: c.hset(_k(i) + "h", mapping=_mk(i))), + ("HGET", lambda c, i: c.hget(_k(i) + "h", "f0")), + ("HDEL", lambda c, i: c.hdel(_k(i) + "hd", "f0")), + ("HEXISTS", lambda c, i: c.hexists(_k(i) + "h", "f0")), + ("HGETALL", lambda c, i: c.hgetall(_k(i) + "h")), + ("HLEN", lambda c, i: c.hlen(_k(i) + "h")), + ("HSTRLEN", lambda c, i: c.hstrlen(_k(i) + "h", "f0")), + ("HINCRBY", lambda c, i: (c.hset(_k(i) + "hc", mapping={"cnt": 0}), + c.hincrby(_k(i) + "hc", "cnt", 1))[1]), + ("HINCRBYFLOAT", lambda c, i: (c.hset(_k(i) + "hf", mapping={"val": "0"}), + c.hincrbyfloat(_k(i) + "hf", "val", 0.5))[1]), + ("HMGET", lambda c, i: c.hmget(_k(i) + "h", ["f0", "f1"])), + ("HKEYS", lambda c, i: c.hkeys(_k(i) + "h")), + ("HVALS", lambda c, i: c.hvals(_k(i) + "h")), + ("HSETNX", lambda c, i: c.hsetnx(_k(i) + "hx", "uniq", Str(i))), + + # ── List ── + ("LPUSH", lambda c, i: c.lpush(_k(i) + "l", Str(i), Str(i + 1))), + ("RPUSH", lambda c, i: c.rpush(_k(i) + "r", Str(i), Str(i + 1))), + ("LPOP", lambda c, i: c.lpop(_k(i) + "l")), + ("RPOP", lambda c, i: c.rpop(_k(i) + "r")), + ("LLEN", lambda c, i: c.llen(_k(i) + "l")), + ("LRANGE", lambda c, i: c.lrange(_k(i) + "l", 0, -1)), + ("LINDEX", lambda c, i: c.lindex(_k(i) + "l", 0)), + ("LSET", lambda c, i: (c.lpush(_k(i) + "ls", "x") or True, + c.lset(_k(i) + "ls", 0, "UPD"))[1]), + ("LTRIM", lambda c, i: c.ltrim(_k(i) + "lt", 0, 1)), + ("LREM", lambda c, i: c.lrem(_k(i) + "l", 1, Str(i))), + ("LPUSHX", lambda c, i: c.lpushx(_k(i) + "l", "xx")), + ("RPUSHX", lambda c, i: c.rpushx(_k(i) + "r", "xx")), + ("RPOPLPUSH", lambda c, i: (c.rpush(_kt(i, "r"), "x") or True, + c.rpoplpush(_kt(i, "r"), _kt(i, "rl")))[1]), + + # ── Set ── + ("SADD", lambda c, i: c.sadd(_k(i) + "s", Str(i), Str(i + 1))), + ("SMEMBERS",lambda c, i: c.smembers(_k(i) + "s")), + ("SCARD", lambda c, i: c.scard(_k(i) + "s")), + ("SISMEMBER",lambda c, i: c.sismember(_k(i) + "s", Str(i))), + ("SMISMEMBER",lambda c, i: c.smismember(_k(i) + "s", [Str(i), Str(i + 1)])), + ("SREM", lambda c, i: c.srem(_k(i) + "s", Str(i))), + ("SPOP", lambda c, i: c.spop(_k(i) + "s")), + ("SRANDMEMBER", lambda c, i: (c.sadd(_kt(i, "s"), "x") or True, + c.srandmember(_kt(i, "s"), 1))[1]), + ("SMOVE", lambda c, i: (c.sadd(_kt(i, "s1"), "x"), c.smove(_kt(i, "s1"), _kt(i, "s2"), "x"))[1]), + ("SUNION", lambda c, i: c.sunion(_kt(i, "s"), _kt(i, "s2"))), + ("SUNIONSTORE", lambda c, i: c.sunionstore(_kt(i, "su"), [_kt(i, "s"), _kt(i, "s2")])), + ("SINTER", lambda c, i: c.sinter(_kt(i, "s"), _kt(i, "s2"))), + ("SINTERCARD", lambda c, i: c.sintercard(2, [_kt(i, "s"), _kt(i, "s2")])), + ("SDIFF", lambda c, i: c.sdiff(_kt(i, "s"), _kt(i, "s2"))), + ("SSCAN", lambda c, i: c.sscan(_k(i) + "s")[1][:1]), + + # ── Sorted Set ── + ("ZADD", lambda c, i: c.zadd(_k(i) + "z", {Str(j): float(j) for j in range(i % 10, i % 10 + 3)})), + ("ZCARD", lambda c, i: c.zcard(_k(i) + "z")), + ("ZCOUNT", lambda c, i: c.zcount(_k(i) + "z", 0, 100)), + ("ZSCORE", lambda c, i: c.zscore(_k(i) + "z", Str(i % 10))), + ("ZMSCORE", lambda c, i: c.zmscore(_k(i) + "z", [Str(i % 10), Str(i % 10 + 1)])), + ("ZRANK", lambda c, i: c.zrank(_k(i) + "z", Str(i % 10))), + ("ZREVRANK",lambda c, i: c.zrevrank(_k(i) + "z", Str(i % 10))), + ("ZRANGE", lambda c, i: c.zrange(_k(i) + "z", 0, -1)), + ("ZRANGEBYSCORE", lambda c, i: c.zrangebyscore(_k(i) + "z", 0, 100)), + ("ZREVRANGE", lambda c, i: c.zrevrange(_k(i) + "z", 0, -1)), + ("ZREVRANGEBYSCORE", lambda c, i: c.zrevrangebyscore(_k(i) + "z", 100, 0)), + ("ZINCRBY", lambda c, i: c.zincrby(_k(i) + "z", 1, Str(i % 10))), + ("ZREM", lambda c, i: c.zrem(_k(i) + "zr", Str(i % 10))), + ("ZREMRANGEBYSCORE", lambda c, i: c.zremrangebyscore(_k(i) + "zr", 0, 50)), + ("ZREMRANGEBYRANK", lambda c, i: c.zremrangebyrank(_k(i) + "zr", 0, 1)), + ("ZLEXCOUNT", lambda c, i: c.zlexcount(_k(i) + "zl", "-", "+")), + ("ZPOPMIN", lambda c, i: c.zpopmin(_k(i) + "z")), + ("ZRANDMEMBER", lambda c, i: c.zrandmember(_k(i) + "z", 1)), + ("ZSCAN", lambda c, i: c.zscan(_k(i) + "z")[1][:1]), + + # ── Generic / Key ── + ("DEL", lambda c, i: (c.set(_k(i) + "td", "x"), c.delete(_k(i) + "td"))[1]), + ("UNLINK", lambda c, i: (c.set(_k(i) + "tu", "x"), c.unlink(_k(i) + "tu"))[1]), + ("EXISTS", lambda c, i: c.exists(_k(i))), + ("TYPE", lambda c, i: c.type(_k(i))), + ("EXPIRE", lambda c, i: (c.set(_k(i) + "te", "x"), c.expire(_k(i) + "te", 300))[1]), + ("PEXPIRE", lambda c, i: (c.set(_k(i) + "tp", "x"), c.pexpire(_k(i) + "tp", 300000))[1]), + ("EXPIREAT",lambda c, i: (c.set(_k(i) + "tea", "x"), c.expireat(_k(i) + "tea", int(time.time()) + 300))[1]), + ("PEXPIREAT",lambda c, i: (c.set(_k(i) + "tpa", "x"), c.pexpireat(_k(i) + "tpa", int(time.time() * 1000) + 300000))[1]), + ("TTL", lambda c, i: c.ttl(_k(i))), + ("PTTL", lambda c, i: c.pttl(_k(i))), + ("PERSIST", lambda c, i: (c.set(_k(i) + "tpe", "x"), c.expire(_k(i) + "tpe", 300), + c.persist(_k(i) + "tpe"))[2]), + ("SCAN", lambda c, i: c.scan(match="stress:cmd:*", count=10)[1][:5]), + ("SORT", lambda c, i: (c.lpush(_k(i) + "so", "2", "1", "3"), c.sort(_k(i) + "so", alpha=True))[1]), + ("KEYS", lambda c, i: c.keys(_k(i) + "*")), + ("DUMP", lambda c, i: c.dump(_k(i))), + ("RESTORE", lambda c, i: (payload := c.dump(_k(i)), + c.restore(_kt(i, "rs"), 0, payload, replace=True))), +] + + +def run_cmd_coverage(client: Redis, tests: List[Tuple[str, Callable[[Redis, int], Any]]]) -> Dict[str, Tuple[int, int, List[str]]]: + """Run every command once and report per-command pass/fail.""" + results: Dict[str, Tuple[int, int, List[str]]] = {} + for name, fn in tests: + ok, fail, errs = 0, 0, [] + try: + fn(client, 0) + ok = 1 + except Exception: + fail = 1 + errs = [traceback.format_exc()[-200:]] + results[name] = (ok, fail, errs) + return results + + +# --------------------------------------------------------------------------- +# Multi-command stress worker +# --------------------------------------------------------------------------- +_CMD_ORDER = [name for name, _ in COMMAND_TESTS] +_CMD_FNS = {name: fn for name, fn in COMMAND_TESTS} + + +def selected_command_tests() -> List[Tuple[str, Callable[[Redis, int], Any]]]: + if args.command_set == "info-only": + return [("INFO", _CMD_FNS["INFO"])] + return COMMAND_TESTS + +def stress_worker(client: Redis, stop_event: threading.Event, phase_event: threading.Event, + stats_lock: threading.Lock, cmd_stats: Dict[str, Dict[str, Any]], + worker_id: int) -> None: + phase_event.wait() + idx = worker_id + key_mod = args.key_count + repeat = args.repeat + while not stop_event.is_set(): + for cmd_name in _CMD_ORDER: + if stop_event.is_set(): + break + fn = _CMD_FNS[cmd_name] + ki = idx % key_mod + for _ in range(repeat): + if stop_event.is_set(): + break + try: + fn(client, ki) + with stats_lock: + cmd_stats[cmd_name]["ok"] += 1 + except Exception as e: + if cmd_name == "ECHO" and "Missing key" in str(e): + continue + with stats_lock: + record_failure(cmd_stats, cmd_name, e) + idx += 1 + + +# --------------------------------------------------------------------------- +# Progress reporter +# --------------------------------------------------------------------------- +def fmt_cmd_stats(cmd_stats: Dict[str, Dict[str, Any]]) -> str: + total_ok = sum(v["ok"] for v in cmd_stats.values()) + total_fail = sum(v["fail"] for v in cmd_stats.values()) + failures = [f"{k}:{v['fail']}" for k, v in cmd_stats.items() if v["fail"] > 0] + s = f"ok={total_ok} fail={total_fail}" + if failures: + s += " | " + " ".join(failures[:10]) + return s + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +def main() -> None: + command_tests = selected_command_tests() + command_order = [name for name, _ in command_tests] + print(f"Command set: {args.command_set} (read_from_replicas={args.read_from_replicas})", flush=True) + + # ── Preload keys (via cluster client to handle MOVED redirects) ── + print(f"Preloading keyspace: key_count={args.key_count}", flush=True) + client = build_cluster_client() + try: + for i in range(args.key_count): + client.set(_k(i), Str(i)) + client.hset(_k(i) + "h", mapping=_mk(i)) + client.lpush(_k(i) + "l", Str(i)) + client.rpush(_k(i) + "r", Str(i)) + client.sadd(_k(i) + "s", Str(i)) + client.zadd(_k(i) + "z", {Str(i % 50): float(i % 50)}) + client.zadd(_k(i) + "zr", {Str(j): float(j) for j in range(i % 5, i % 5 + 3)}) + client.zadd(_k(i) + "zl", {f"a{j}": 0 for j in range(3)}) + client.sadd(_k(i) + "s2", "shared") + client.lpush(_k(i) + "lt", "a", "b", "c") + finally: + close_client(client) + print("Keyspace ready.", flush=True) + + # ── Quick command coverage check (standalone + cluster) ── + if not args.skip_cmd_coverage: + print("Running command coverage check (standalone) ...", flush=True) + coverage = run_cmd_coverage(client := build_client(master_node), command_tests) + close_client(client) + total_ok = sum(v[0] for v in coverage.values()) + total_fail = sum(v[1] for v in coverage.values()) + failures = [(n, errs) for n, (ok, fail, errs) in coverage.items() if fail] + print(f" standalone coverage: {total_ok}/{total_ok + total_fail}", flush=True) + for name, errs in failures: + print(f" FAIL {name}: {errs[0][:120] if errs else ''}", flush=True) + + print("Running command coverage check (cluster) ...", flush=True) + ccoverage = run_cmd_coverage(client := build_cluster_client(), command_tests) + close_client(client) + ctotal_ok = sum(v[0] for v in ccoverage.values()) + ctotal_fail = sum(v[1] for v in ccoverage.values()) + cfailures = [(n, errs) for n, (ok, fail, errs) in ccoverage.items() if fail] + print(f" cluster coverage: {ctotal_ok}/{ctotal_ok + ctotal_fail}", flush=True) + for name, errs in cfailures: + print(f" FAIL {name}: {errs[0][:120] if errs else ''}", flush=True) + + stop_event = threading.Event() + + # ── Build clients: standalone + cluster ── + standalone_client = build_client(master_node) + cluster_client = build_cluster_client() + + workers: List[Tuple[Redis, threading.Thread]] = [] + standalone_stats = { + name: {"ok": 0, "fail": 0, "error_types": Counter(), "samples": []} + for name in command_order + } + cluster_stats = { + name: {"ok": 0, "fail": 0, "error_types": Counter(), "samples": []} + for name in command_order + } + stats_lock = threading.Lock() + phase_event = threading.Event() + + n_standalone = args.workers // 2 + n_cluster = args.workers - n_standalone + total_standalone_slots = n_standalone * args.inflight + total_cluster_slots = n_cluster * args.inflight + print( + f"Starting {n_standalone} standalone + {n_cluster} cluster workers " + f"with inflight={args.inflight} " + f"({total_standalone_slots} + {total_cluster_slots} execution slots) ...", + flush=True, + ) + for i in range(total_standalone_slots): + cli = standalone_client + th = threading.Thread(target=stress_worker, + args=(cli, stop_event, phase_event, + stats_lock, standalone_stats, i), + daemon=True) + th.start() + workers.append((cli, th)) + for i in range(total_cluster_slots): + cli = cluster_client + th = threading.Thread(target=stress_worker, + args=(cli, stop_event, phase_event, + stats_lock, cluster_stats, i + total_standalone_slots), + daemon=True) + th.start() + workers.append((cli, th)) + + phase_event.set() + + # ── Progress loop ── + deadline = (time.time() + args.duration) if args.duration > 0 else float("inf") + + def _should_stop(): + return stop_event.is_set() or time.time() >= deadline + + try: + last = time.time() + while not _should_stop(): + time.sleep(args.progress_interval) + now = time.time() + elapsed = now - last + s_ok = sum(v["ok"] for v in standalone_stats.values()) + s_fail = sum(v["fail"] for v in standalone_stats.values()) + c_ok = sum(v["ok"] for v in cluster_stats.values()) + c_fail = sum(v["fail"] for v in cluster_stats.values()) + total_ok = s_ok + c_ok + total_fail = s_fail + c_fail + qps = (total_ok + total_fail) / max(elapsed, 0.001) + print(f"[progress] standalone: ok={s_ok} fail={s_fail} " + f"cluster: ok={c_ok} fail={c_fail} " + f"total_qps={qps:.0f}", flush=True) + except KeyboardInterrupt: + pass + + stop_event.set() + for cli, th in workers: + th.join(timeout=3) + close_client(standalone_client) + close_client(cluster_client) + + # ── Report ── + def _report(title: str, stats: Dict[str, Dict[str, Any]]) -> None: + total_ok = sum(v["ok"] for v in stats.values()) + total_fail = sum(v["fail"] for v in stats.values()) + print(f"\n--- {title} ---") + print(f"total_commands: ok={total_ok} fail={total_fail}") + for name in command_order: + ok, fail = stats[name]["ok"], stats[name]["fail"] + if fail > 0: + print(f" {name}: ok={ok} fail={fail}") + for signature, count in stats[name]["error_types"].most_common(ERROR_TYPE_LIMIT): + print(f" error[{count}]: {signature[:200]}") + for sample in stats[name]["samples"]: + print(f" sample: {sample[:200]}") + _report("Standalone Client Results", standalone_stats) + _report("Cluster Client Results", cluster_stats) + s_fail = sum(v["fail"] for v in standalone_stats.values()) + c_fail = sum(v["fail"] for v in cluster_stats.values()) + s_ok = sum(v["ok"] for v in standalone_stats.values()) + c_ok = sum(v["ok"] for v in cluster_stats.values()) + total_ok = s_ok + c_ok + total_fail = s_fail + c_fail + if args.results_file: + with open(args.results_file, "w") as f: + json.dump({ + "standalone": { + name: { + "ok": data["ok"], + "fail": data["fail"], + "error_types": dict(data["error_types"]), + "samples": data["samples"], + } + for name, data in standalone_stats.items() + }, + "cluster": { + name: { + "ok": data["ok"], + "fail": data["fail"], + "error_types": dict(data["error_types"]), + "samples": data["samples"], + } + for name, data in cluster_stats.items() + }, + }, f, indent=2) + # Tolerate <2% sporadic failures (race conditions from concurrent key access) + if total_fail > 0 and (total_ok == 0 or total_fail / (total_ok + total_fail) > 0.02): + sys.exit(1) + + +if __name__ == "__main__": + _SELECTED_COMMAND_TESTS = selected_command_tests() + _CMD_ORDER = [name for name, _ in _SELECTED_COMMAND_TESTS] + _CMD_FNS = {name: fn for name, fn in _SELECTED_COMMAND_TESTS} + main() diff --git a/tests/e2e/cmd_stress_py/requirements.txt b/tests/e2e/cmd_stress_py/requirements.txt new file mode 100644 index 00000000..785cc924 --- /dev/null +++ b/tests/e2e/cmd_stress_py/requirements.txt @@ -0,0 +1 @@ +redis==5.0.1 diff --git a/tests/e2e/cmd_stress_test.sh b/tests/e2e/cmd_stress_test.sh new file mode 100755 index 00000000..4e268fa6 --- /dev/null +++ b/tests/e2e/cmd_stress_test.sh @@ -0,0 +1,343 @@ +#!/bin/bash +# E2E multi-language command stress test for eloqkv cluster. +# +# Each language SDK runs inside its own Docker container (same subnet as cluster). +# - Python → stress-client (redis-py) +# - Go → stress-go (go-redis/v9) +# - TS → stress-ts (ioredis) +# +# Env overrides: +# STEPS=launch,eloqctl-mutate,py-stress,go-stress,ts-stress,remove +# DURATION_SECONDS=300 WORKERS=8 INFLIGHT=4 KEY_COUNT=256 REPEAT=10 +# CMD_TIMEOUT=5 PROGRESS_INTERVAL=5 +# TLS_ENABLED=1 SKIP_DEPS=1 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../" && pwd)" +source "${REPO_ROOT}/tests/docker_env.sh" + +CLUSTER="test-e2e" +TOPO="${SCRIPT_DIR}/topology.generated.yaml" +CONTROL_TOPO="${CONTROL_REPO_ROOT}/tests/e2e/topology.generated.yaml" +STEPS="${STEPS:-launch,eloqctl-mutate,py-stress,go-stress,ts-stress,remove}" + +DURATION="${DURATION_SECONDS:-300}" +WORKERS="${WORKERS:-16}" +INFLIGHT="${INFLIGHT:-4}" +REPEAT="${REPEAT:-10}" +KEY_COUNT="${KEY_COUNT:-256}" +CMD_TIMEOUT="${CMD_TIMEOUT:-5}" +PROGRESS_INTERVAL="${PROGRESS_INTERVAL:-5}" +TLS_ENABLED="${TLS_ENABLED:-1}" +SKIP_DEPS="${SKIP_DEPS:-1}" +INFO_ONLY_WORKERS="${INFO_ONLY_WORKERS:-64}" +INFO_ONLY_INFLIGHT="${INFO_ONLY_INFLIGHT:-16}" +INFO_ONLY_REPEAT="${INFO_ONLY_REPEAT:-50}" +INFO_ONLY_DURATION="${INFO_ONLY_DURATION_SECONDS:-300}" + +PASSWD="testpass" +N1="172.28.10.11" +N2="172.28.10.12" +STARTUP_NODES="${N1}:6379,${N2}:6379" +MASTER="" # discovered dynamically +REPLICA="" + +# Accumulated failures across steps +FAILED_STEPS="" + +TLS_FLAG="" +[ "${TLS_ENABLED}" = "1" ] && TLS_FLAG="--tls" + +control_eloqctl_cmd() { + control_exec env ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" "${CONTROL_ELOQCTL}" "$@" +} + +run_control_eloqctl_with_progress() { + local timeout_seconds="$1" + local log_file="$2" + shift 2 + run_with_progress "${timeout_seconds}" "${log_file}" \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ + "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ + "${CONTROL_ELOQCTL}" "$@" \ + || { dump_failure_diagnostics "${log_file}"; return 1; } +} + +wait_cluster_ready() { + run_with_progress 240 "${SCRIPT_DIR}/cmd-stress-status.log" \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ + "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ + "${CONTROL_ELOQCTL}" status "${CLUSTER}" --wait 180 >/dev/null 2>&1 \ + || { echo "FAIL: cluster not healthy"; return 1; } +} + +cleanup() { + if [ "${KEEP_LOGS:-0}" != "1" ]; then + rm -f "${SCRIPT_DIR}/cmd-stress-"*.log "${SCRIPT_DIR}/launch-cmd-stress.log" "${TOPO}" + fi + if [ -n "${FAILED_STEPS}" ]; then + echo "" + echo "FAILED STEPS: ${FAILED_STEPS}" + exit 1 + fi +} +trap cleanup EXIT + +step() { + local name="$1"; shift + if [[ ",${STEPS}," == *",${name},"* ]]; then + echo "=== step: ${name} ===" + if "$@"; then + echo " ${name} PASS" + else + echo " ${name} FAIL" + FAILED_STEPS="${FAILED_STEPS} ${name}" + fi + else + echo "[skip] ${name}" + fi +} + +# ── Master discovery (works independently of launch) ── +discover_master() { + echo " discovering cluster topology ..." + local nodes_info + nodes_info=$(compose exec -T stress-client python3 -c " +import ssl +TLS={'ssl':True,'ssl_cert_reqs':ssl.CERT_NONE,'ssl_check_hostname':False} +from redis import Redis +r=Redis(host='${N1}',port=6379,password='${PASSWD}',socket_timeout=5,**TLS) +print(r.execute_command('CLUSTER','NODES').decode()) +r.close() +" 2>/dev/null) || { echo "FAIL: cannot connect to cluster for discovery"; return 1; } + + MASTER="" + while IFS= read -r line; do + local addr role + addr=$(echo "$line" | awk '{print $2}' | cut -d@ -f1) + role=$(echo "$line" | awk '{print $3}') + if echo "$role" | grep -q 'master'; then + MASTER="${addr}" + elif echo "$role" | grep -q 'slave'; then + REPLICA="${addr}" + fi + done <<< "$nodes_info" + if [ -z "$MASTER" ]; then + echo "FAIL: could not discover master from CLUSTER NODES" + return 1 + fi + echo " master=${MASTER} replica=${REPLICA}" + sleep 5 + return 0 +} + +# ── Launch ── +do_launch() { + echo "=== Launch cluster ===" + local launch_args=() + if [ "${SKIP_DEPS}" = "1" ]; then + launch_args+=(--skip-deps) + fi + render_topology_for_control "${SCRIPT_DIR}/topology.yaml" "${TOPO}" + start_docker_env + control_eloqctl_cmd stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true + control_eloqctl_cmd remove "${CLUSTER}" --force >/dev/null 2>&1 || true + run_with_progress 420 "${SCRIPT_DIR}/launch-cmd-stress.log" \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ + "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ + "${CONTROL_ELOQCTL}" launch "${launch_args[@]}" "${CONTROL_TOPO}" \ + || { dump_failure_diagnostics "${SCRIPT_DIR}/launch-cmd-stress.log"; return 1; } + run_with_progress 240 "${SCRIPT_DIR}/launch-cmd-stress.log" \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ + "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ + "${CONTROL_ELOQCTL}" status "${CLUSTER}" --wait 180 >/dev/null 2>&1 \ + || { echo "FAIL: cluster not healthy after launch"; return 1; } + echo " cluster ready" + discover_master || return 1 +} + +do_eloqctl_mutate() { + local original_master original_replica + local original_master_host original_master_port + local original_replica_host original_replica_port + local current_master_host current_master_port + + discover_master || return 1 + original_master="${MASTER}" + original_replica="${REPLICA}" + if [ -z "${original_replica}" ]; then + echo "FAIL: no replica discovered for failover test" + return 1 + fi + + original_master_host="${original_master%:*}" + original_master_port="${original_master##*:}" + original_replica_host="${original_replica%:*}" + original_replica_port="${original_replica##*:}" + + echo "=== eloqctl mutation check ===" + + echo " failover ${original_master} -> ${original_replica}" + run_control_eloqctl_with_progress 240 "${SCRIPT_DIR}/cmd-stress-failover-1.log" \ + failover "${CLUSTER}" \ + --old-leader-host "${original_master_host}" --old-leader-port "${original_master_port}" \ + --new-leader-host "${original_replica_host}" --new-leader-port "${original_replica_port}" \ + --password "${PASSWD}" \ + || return 1 + wait_cluster_ready || return 1 + discover_master || return 1 + if [ "${MASTER}" != "${original_replica}" ]; then + echo "FAIL: failover did not move master to ${original_replica}; current master=${MASTER}" + return 1 + fi + + current_master_host="${MASTER%:*}" + current_master_port="${MASTER##*:}" + echo " failover ${MASTER} -> ${original_master}" + run_control_eloqctl_with_progress 240 "${SCRIPT_DIR}/cmd-stress-failover-2.log" \ + failover "${CLUSTER}" \ + --old-leader-host "${current_master_host}" --old-leader-port "${current_master_port}" \ + --new-leader-host "${original_master_host}" --new-leader-port "${original_master_port}" \ + --password "${PASSWD}" \ + || return 1 + wait_cluster_ready || return 1 + discover_master || return 1 + if [ "${MASTER}" != "${original_master}" ]; then + echo "FAIL: failback did not restore master to ${original_master}; current master=${MASTER}" + return 1 + fi + + echo " eloqctl mutations verified" +} + +# ── Python stress (inside stress-client container) ── +do_py_stress() { + discover_master || return 1 + echo "=== Python command stress (redis-py) ===" + local py_startup_args=(--startup-node "${MASTER}") + if [ -n "${REPLICA}" ]; then + py_startup_args+=(--startup-node "${REPLICA}") + fi + compose exec -T stress-client python3 -u tests/e2e/cmd_stress_py/main.py \ + "${py_startup_args[@]}" \ + --password "${PASSWD}" --cmd-timeout "${CMD_TIMEOUT}" \ + --progress-interval "${PROGRESS_INTERVAL}" --key-count "${KEY_COUNT}" \ + --workers "${WORKERS}" --inflight "${INFLIGHT}" --duration "${DURATION}" --repeat "${REPEAT}" \ + ${TLS_FLAG} 2>&1 | tee "${SCRIPT_DIR}/cmd-stress-py.log" + local py_status=${PIPESTATUS[0]} + [ ${py_status} -eq 0 ] || return ${py_status} + + echo "=== Python INFO burst (redis-py) ===" + compose exec -T stress-client python3 -u tests/e2e/cmd_stress_py/main.py \ + "${py_startup_args[@]}" \ + --password "${PASSWD}" --cmd-timeout "${CMD_TIMEOUT}" \ + --progress-interval "${PROGRESS_INTERVAL}" --key-count "${KEY_COUNT}" \ + --workers "${INFO_ONLY_WORKERS}" --inflight "${INFO_ONLY_INFLIGHT}" \ + --duration "${INFO_ONLY_DURATION}" --repeat "${INFO_ONLY_REPEAT}" \ + --command-set info-only \ + ${TLS_FLAG} 2>&1 | tee "${SCRIPT_DIR}/cmd-stress-py-info.log" + return ${PIPESTATUS[0]} +} + +# ── Go stress (inside stress-go container) ── +do_go_stress() { + echo "=== Go command stress (go-redis/v9) ===" + echo " installing Go deps ..." + compose exec -T stress-go bash -c \ + 'cd tests/e2e/cmd_stress_go && go mod download' 2>&1 || true + compose exec -T stress-go bash -c \ + "cd tests/e2e/cmd_stress_go && go run . \ + --startup-nodes '${STARTUP_NODES}' \ + --password '${PASSWD}' \ + --workers ${WORKERS} \ + --inflight ${INFLIGHT} \ + --duration ${DURATION}s \ + --repeat ${REPEAT} \ + --progress-interval ${PROGRESS_INTERVAL}s \ + --key-count ${KEY_COUNT} \ + --cmd-timeout ${CMD_TIMEOUT}s \ + $([ "${TLS_ENABLED}" = "1" ] && echo '--tls-insecure')" \ + 2>&1 | tee "${SCRIPT_DIR}/cmd-stress-go.log" + local go_status=${PIPESTATUS[0]} + [ ${go_status} -eq 0 ] || return ${go_status} + + echo "=== Go INFO burst (go-redis/v9) ===" + compose exec -T stress-go bash -c \ + "cd tests/e2e/cmd_stress_go && go run . \ + --startup-nodes '${STARTUP_NODES}' \ + --password '${PASSWD}' \ + --workers ${INFO_ONLY_WORKERS} \ + --inflight ${INFO_ONLY_INFLIGHT} \ + --duration ${INFO_ONLY_DURATION}s \ + --repeat ${INFO_ONLY_REPEAT} \ + --progress-interval ${PROGRESS_INTERVAL}s \ + --key-count ${KEY_COUNT} \ + --cmd-timeout ${CMD_TIMEOUT}s \ + --command-set info-only \ + $([ "${TLS_ENABLED}" = "1" ] && echo '--tls-insecure')" \ + 2>&1 | tee "${SCRIPT_DIR}/cmd-stress-go-info.log" + return ${PIPESTATUS[0]} +} + +# ── TypeScript stress (inside stress-ts container) ── +do_ts_stress() { + echo "=== TypeScript command stress (ioredis) ===" + echo " installing npm deps ..." + compose exec -T stress-ts bash -c \ + 'cd tests/e2e/cmd_stress_ts && npm install --silent' 2>&1 || true + compose exec -T stress-ts bash -c \ + "cd tests/e2e/cmd_stress_ts && npx tsx main.ts \ + --startup-nodes '${STARTUP_NODES}' \ + --password '${PASSWD}' \ + --workers ${WORKERS} \ + --inflight ${INFLIGHT} \ + --duration ${DURATION} \ + --repeat ${REPEAT} \ + --progress-interval ${PROGRESS_INTERVAL} \ + --key-count ${KEY_COUNT} \ + --cmd-timeout ${CMD_TIMEOUT} \ + --read-from-replicas 'false' \ + --tls-insecure '${TLS_ENABLED}'" \ + 2>&1 | tee "${SCRIPT_DIR}/cmd-stress-ts.log" + local ts_status=${PIPESTATUS[0]} + [ ${ts_status} -eq 0 ] || return ${ts_status} + + echo "=== TypeScript INFO burst (ioredis) ===" + compose exec -T stress-ts bash -c \ + "cd tests/e2e/cmd_stress_ts && npx tsx main.ts \ + --startup-nodes '${STARTUP_NODES}' \ + --password '${PASSWD}' \ + --workers ${INFO_ONLY_WORKERS} \ + --inflight ${INFO_ONLY_INFLIGHT} \ + --duration ${INFO_ONLY_DURATION} \ + --repeat ${INFO_ONLY_REPEAT} \ + --progress-interval ${PROGRESS_INTERVAL} \ + --key-count ${KEY_COUNT} \ + --cmd-timeout ${CMD_TIMEOUT} \ + --command-set info-only \ + --read-from-replicas 'false' \ + --tls-insecure '${TLS_ENABLED}'" \ + 2>&1 | tee "${SCRIPT_DIR}/cmd-stress-ts-info.log" + return ${PIPESTATUS[0]} +} + +# ── Remove ── +do_remove() { + echo "=== Remove cluster ===" + control_eloqctl_cmd stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true + control_eloqctl_cmd remove "${CLUSTER}" --force >/dev/null 2>&1 || true + compose_down + echo " removed" +} + +step launch do_launch +step eloqctl-mutate do_eloqctl_mutate +step py-stress do_py_stress +step go-stress do_go_stress +step ts-stress do_ts_stress +step remove do_remove + +if [ -z "${FAILED_STEPS}" ]; then + echo "ALL PASS" +fi diff --git a/tests/e2e/cmd_stress_ts/main.ts b/tests/e2e/cmd_stress_ts/main.ts new file mode 100644 index 00000000..b64405bd --- /dev/null +++ b/tests/e2e/cmd_stress_ts/main.ts @@ -0,0 +1,469 @@ +#!/usr/bin/env npx tsx +/** + * eloqkv full-command stress test — TypeScript / ioredis SDK + * + * Covers Redis commands across all supported families. + */ +import { Cluster, ClusterNode, Redis } from "ioredis"; +import { parseArgs } from "node:util"; +import * as path from "node:path"; + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- +const { values: args } = parseArgs({ + args: process.argv.slice(2), + options: { + "startup-nodes": { type: "string", default: "127.0.0.1:6379,127.0.0.1:6380" }, + password: { type: "string", default: "testpass" }, + "cmd-timeout": { type: "string", default: "5000" }, + "progress-interval": { type: "string", default: "5000" }, + "key-count": { type: "string", default: "256" }, + workers: { type: "string", default: "16" }, + inflight: { type: "string", default: "4" }, + repeat: { type: "string", default: "10" }, + duration: { type: "string", default: "0" }, + "tls-insecure": { type: "string", default: "true" }, + "read-from-replicas": { type: "string", default: "true" }, + "command-set": { type: "string", default: "full" }, + "log-prefix": { type: "string", default: "" }, + }, +}); + +const startupNodes = args["startup-nodes"]!.split(",").map(s => s.trim()); +const password = args.password!; +const CMD_TIMEOUT = parseInt(args["cmd-timeout"]!) * 1000; +const PROGRESS_INTERVAL = parseInt(args["progress-interval"]!) * 1000; +const KEY_COUNT = parseInt(args["key-count"]!); +const WORKERS = parseInt(args.workers!); +const INFLIGHT = parseInt(args.inflight!); +const REPEAT = parseInt(args.repeat!); +const DURATION = parseInt(args.duration!); +const TLS_INSECURE = args["tls-insecure"] !== "false"; +const READ_FROM_REPLICAS = args["read-from-replicas"] !== "false"; +const COMMAND_SET = args["command-set"] === "info-only" ? "info-only" : "full"; +const LOG_PREFIX = args["log-prefix"] || ""; + +const P = (s: string) => (LOG_PREFIX ? `[${LOG_PREFIX}] ${s}` : s); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- +function k(i: number) { return `stress:cmd:${i}`; } +function kt(i: number, suffix = "") { return `{stress:cmd:${i}}${suffix}`; } + +function mkClient(addr: string): Redis { + const [host, portStr] = addr.split(":"); + const port = parseInt(portStr || "6379"); + const tlsOpts = TLS_INSECURE ? { tls: { rejectUnauthorized: false } as any } : {}; + return new Redis({ + host, port, password, + connectTimeout: CMD_TIMEOUT, + commandTimeout: CMD_TIMEOUT, + maxRetriesPerRequest: 1, + maxLoadingRetryTime: 1000, + family: 4, + connectionName: "cmd-stress", + retryStrategy: () => null, + lazyConnect: false, + ...tlsOpts, + }); +} + +// --------------------------------------------------------------------------- +// Command test definitions +// Each entry: [name, fn(client, idx) => Promise] +// --------------------------------------------------------------------------- +type CmdFn = (c: any, idx: number) => Promise; + +const CMD_TESTS: [string, CmdFn][] = [ + // ── Connection / Server ── + ["PING", (c) => c.ping()], + ["ECHO", (c, i) => c.echo(String(i))], + ["DBSIZE", (c) => c.dbsize()], + ["TIME", (c) => c.time()], + ["INFO", (c) => c.info()], + ["CLUSTER_INFO", (c) => c.call("CLUSTER", "INFO")], + + // ── String ── + ["SET", (c, i) => c.set(k(i), String(i))], + ["GET", (c, i) => c.get(k(i))], + ["GETDEL", async (c, i) => { await c.set(k(i) + "gd", "x"); return c.getdel(k(i) + "gd"); }], + ["SETNX", (c, i) => c.setnx(k(i) + "nx", String(i))], + ["SETEX", (c, i) => c.setex(k(i) + "ex", 60, String(i))], + ["PSETEX", (c, i) => c.psetex(k(i) + "px", 60000, String(i))], + ["STRLEN", (c, i) => c.strlen(k(i))], + ["GETRANGE", (c, i) => c.getrange(k(i), 0, 3)], + ["SETRANGE", (c, i) => c.setrange(k(i) + "sr", 0, "xx")], + ["APPEND", (c, i) => c.append(k(i) + "ap", "tail")], + ["GETBIT", (c, i) => c.getbit(k(i), 0)], + ["SETBIT", (c, i) => c.setbit(k(i) + "bi", 7, 1)], + ["BITCOUNT", (c, i) => c.bitcount(k(i))], + ["BITPOS", (c, i) => c.bitpos(k(i), 1)], + ["INCR", (c, i) => c.incr(k(i) + "ctr")], + ["DECR", (c, i) => c.decr(k(i) + "ctr")], + ["INCRBY", (c, i) => c.incrby(k(i) + "ctr", 10)], + ["DECRBY", (c, i) => c.decrby(k(i) + "ctr", 5)], + ["INCRBYFLOAT", (c, i) => c.incrbyfloat(k(i) + "fl", 0.5)], + ["MSET", (c, i) => c.mset(kt(i, "m0"), "a", kt(i, "m1"), "b")], + ["MGET", (c, i) => c.mget(kt(i, "m0"), kt(i, "m1"))], + + // ── Hash ── + ["HSET", (c, i) => c.hset(k(i) + "h", { f0: "v0", f1: "v1" })], + ["HGET", (c, i) => c.hget(k(i) + "h", "f0")], + ["HDEL", async (c, i) => { await c.hset(k(i) + "hd", "f0", "x"); return c.hdel(k(i) + "hd", "f0"); }], + ["HEXISTS", (c, i) => c.hexists(k(i) + "h", "f0")], + ["HGETALL", (c, i) => c.hgetall(k(i) + "h")], + ["HLEN", (c, i) => c.hlen(k(i) + "h")], + ["HSTRLEN", (c, i) => c.hstrlen(k(i) + "h", "f0")], + ["HINCRBY", async (c, i) => { await c.hset(k(i) + "hc", "cnt", 0); return c.hincrby(k(i) + "hc", "cnt", 1); }], + ["HMGET", (c, i) => c.hmget(k(i) + "h", "f0", "f1")], + ["HINCRBYFLOAT", async (c, i) => { await c.hset(k(i) + "hf", "val", "0"); return c.hincrbyfloat(k(i) + "hf", "val", 0.5); }], + ["HKEYS", (c, i) => c.hkeys(k(i) + "h")], + ["HVALS", (c, i) => c.hvals(k(i) + "h")], + ["HSETNX", (c, i) => c.hsetnx(k(i) + "hx", "uniq", String(i))], + + // ── List ── + ["LPUSH", (c, i) => c.lpush(k(i) + "l", String(i), String(i + 1))], + ["RPUSH", (c, i) => c.rpush(k(i) + "r", String(i), String(i + 1))], + ["LPOP", (c, i) => c.lpop(k(i) + "l")], + ["RPOP", (c, i) => c.rpop(k(i) + "r")], + ["LLEN", (c, i) => c.llen(k(i) + "l")], + ["LRANGE", (c, i) => c.lrange(k(i) + "l", 0, -1)], + ["LINDEX", (c, i) => c.lindex(k(i) + "l", 0)], + ["LSET", async (c, i) => { await c.lpush(k(i) + "ls", "x"); return c.lset(k(i) + "ls", 0, "UPD"); }], + ["LTRIM", (c, i) => c.ltrim(k(i) + "lt", 0, 1)], + ["LREM", (c, i) => c.lrem(k(i) + "l", 1, String(i))], + ["LPUSHX", (c, i) => c.lpushx(k(i) + "l", "xx")], + ["RPUSHX", (c, i) => c.rpushx(k(i) + "r", "xx")], + ["RPOPLPUSH", async (c, i) => { await c.rpush(kt(i, "r"), "x"); return c.rpoplpush(kt(i, "r"), kt(i, "rl")); }], + + // ── Set ── + ["SADD", (c, i) => c.sadd(k(i) + "s", String(i), String(i + 1))], + ["SMEMBERS", (c, i) => c.smembers(k(i) + "s")], + ["SCARD", (c, i) => c.scard(k(i) + "s")], + ["SISMEMBER", (c, i) => c.sismember(k(i) + "s", String(i))], + ["SREM", (c, i) => c.srem(k(i) + "s", String(i))], + ["SPOP", (c, i) => c.spop(k(i) + "s")], + ["SRANDMEMBER", async (c, i) => { await c.sadd(kt(i, "s"), "x"); return c.srandmember(kt(i, "s")); }], + ["SMOVE", async (c, i) => { await c.sadd(kt(i, "s1"), "x"); return c.smove(kt(i, "s1"), kt(i, "s2"), "x"); }], + ["SUNION", (c, i) => c.sunion(kt(i, "s"), kt(i, "s2"))], + ["SUNIONSTORE", (c, i) => c.sunionstore(kt(i, "su"), kt(i, "s"), kt(i, "s2"))], + ["SINTER", (c, i) => c.sinter(kt(i, "s"), kt(i, "s2"))], + ["SDIFF", (c, i) => c.sdiff(kt(i, "s"), kt(i, "s2"))], + ["SMISMEMBER", (c, i) => c.smismember(kt(i, "s"), String(i), String(i + 1))], + ["SSCAN", (c, i) => c.sscan(k(i) + "s", 0)], + + // ── Sorted Set ── + ["ZADD", (c, i) => c.zadd(k(i) + "z", i % 50, String(i % 50))], + ["ZCARD", (c, i) => c.zcard(k(i) + "z")], + ["ZCOUNT", (c, i) => c.zcount(k(i) + "z", "0", "100")], + ["ZSCORE", (c, i) => c.zscore(k(i) + "z", String(i % 50))], + ["ZRANK", (c, i) => c.zrank(k(i) + "z", String(i % 50))], + ["ZREVRANK", (c, i) => c.zrevrank(k(i) + "z", String(i % 50))], + ["ZRANGE", (c, i) => c.zrange(k(i) + "z", 0, -1)], + ["ZRANGEBYSCORE", (c, i) => c.zrangebyscore(k(i) + "z", "0", "100")], + ["ZREVRANGE", (c, i) => c.zrevrange(k(i) + "z", 0, -1)], + ["ZINCRBY", (c, i) => c.zincrby(k(i) + "z", 1, String(i % 50))], + ["ZREM", (c, i) => c.zrem(k(i) + "zr", String(i % 5))], + ["ZREMRANGEBYSCORE", (c, i) => c.zremrangebyscore(k(i) + "zr", "0", "50")], + ["ZREMRANGEBYRANK", (c, i) => c.zremrangebyrank(k(i) + "zr", 0, 1)], + ["ZPOPMIN", (c, i) => c.zpopmin(k(i) + "z")], + ["ZRANDMEMBER", (c, i) => c.zrandmember(k(i) + "z", 1)], + ["ZMSCORE", (c, i) => c.zmscore(k(i) + "z", String(i % 50), String(i % 50 + 1))], + ["ZLEXCOUNT", (c, i) => c.zlexcount(k(i) + "zl", "-", "+")], + ["ZSCAN", (c, i) => c.zscan(k(i) + "z", 0)], + + // ── Generic / Key ── + ["DEL", async (c, i) => { await c.set(k(i) + "td", "x"); return c.del(k(i) + "td"); }], + ["UNLINK", async (c, i) => { await c.set(k(i) + "tu", "x"); return c.unlink(k(i) + "tu"); }], + ["EXISTS", (c, i) => c.exists(k(i))], + ["TYPE", (c, i) => c.type(k(i))], + ["EXPIRE", async (c, i) => { await c.set(k(i) + "te", "x"); return c.expire(k(i) + "te", 300); }], + ["PEXPIRE", async (c, i) => { await c.set(k(i) + "tp", "x"); return c.pexpire(k(i) + "tp", 300000); }], + ["EXPIREAT", async (c, i) => { await c.set(k(i) + "tea", "x"); return c.expireat(k(i) + "tea", Math.floor(Date.now() / 1000) + 300); }], + ["PEXPIREAT", async (c, i) => { await c.set(k(i) + "tpa", "x"); return c.pexpireat(k(i) + "tpa", Date.now() + 300000); }], + ["TTL", (c, i) => c.ttl(k(i))], + ["PTTL", (c, i) => c.pttl(k(i))], + ["PERSIST", async (c, i) => { + await c.set(k(i) + "tpe", "x"); + await c.expire(k(i) + "tpe", 300); + return c.persist(k(i) + "tpe"); + }], + ["KEYS", (c, i) => c.keys(k(i) + "*")], + ["SCAN", (c) => c.scan(0, "MATCH", "stress:cmd:*", "COUNT", 10)], + ["SORT", async (c, i) => { await c.lpush(k(i) + "so", "2", "1", "3"); return c.sort(k(i) + "so", "ALPHA"); }], + ["DUMP", (c, i) => c.dump(k(i))], + ["RESTORE", async (c, i) => { + try { + const payload = await c.dump(k(i)) as string; + if (!payload) throw new Error("empty payload"); + return c.call("RESTORE", kt(i, "rs"), 0, payload, "REPLACE"); + } catch { + return "skip"; // DUMP binary payloads not reliably round-tripped by ioredis + } + }], +]; + +const SELECTED_CMD_TESTS: [string, CmdFn][] = COMMAND_SET === "info-only" + ? CMD_TESTS.filter(([name]) => name === "INFO") + : CMD_TESTS; + +// --------------------------------------------------------------------------- +// Stats +// --------------------------------------------------------------------------- +class CmdStats { + private counts = new Map; + samples: string[]; + }>(); + private get(name: string) { + let v = this.counts.get(name); + if (!v) { + v = { ok: 0, fail: 0, errorTypes: new Map(), samples: [] }; + this.counts.set(name, v); + } + return v; + } + addOK(name: string) { + const v = this.get(name); + v.ok++; + } + addFail(name: string, err: unknown) { + const v = this.get(name); + v.fail++; + const signature = errorSignature(err); + v.errorTypes.set(signature, (v.errorTypes.get(signature) || 0) + 1); + if (v.samples.length < 3 && !v.samples.includes(signature)) v.samples.push(signature); + } + snapshot(): Map; + samples: string[]; + }> { + return new Map(Array.from(this.counts.entries(), ([name, value]) => [name, { + ok: value.ok, + fail: value.fail, + errorTypes: new Map(value.errorTypes), + samples: [...value.samples], + }])); + } +} + +function errorSignature(err: unknown): string { + if (err instanceof Error) { + const message = err.message.replace(/\s+/g, " ").trim(); + return `${err.name}: ${message || err.toString()}`; + } + const message = String(err).replace(/\s+/g, " ").trim(); + return `Error: ${message}`; +} + +function topErrorCounts(errorTypes: Map): [string, number][] { + return Array.from(errorTypes.entries()) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, 5); +} + +// --------------------------------------------------------------------------- +// Worker +// --------------------------------------------------------------------------- +async function stressWorker( + client: any, tests: [string, CmdFn][], stats: CmdStats, stop: AbortSignal, + startKeyIdx: number, keyMod: number, repeat: number, +): Promise { + let cmdIdx = 0; + let keyIdx = startKeyIdx; + while (!stop.aborted) { + const [name, fn] = tests[cmdIdx % tests.length]; + const ki = keyIdx % keyMod; + for (let r = 0; r < repeat && !stop.aborted; r++) { + try { + await fn(client, ki); + stats.addOK(name); + } catch (err) { + stats.addFail(name, err); + } + } + cmdIdx++; + if (cmdIdx % tests.length === 0) keyIdx++; + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +async function main() { + console.log(P(`starting eloqkv command stress test (TypeScript/ioredis)`)); + console.log(P(`nodes=${startupNodes} workers=${WORKERS} inflight=${INFLIGHT} duration=${DURATION}s key_count=${KEY_COUNT} command_set=${COMMAND_SET} read_from_replicas=${READ_FROM_REPLICAS}`)); + + // Cluster discovery + const cluster = new Cluster(startupNodes.map(addr => { + const [host, portStr] = addr.split(":"); + return { host, port: parseInt(portStr || "6379") }; + }), { + scaleReads: READ_FROM_REPLICAS ? "slave" : "master", + slotsRefreshTimeout: CMD_TIMEOUT, + redisOptions: { + password, + connectTimeout: CMD_TIMEOUT, + commandTimeout: CMD_TIMEOUT, + retryStrategy: () => null, + ...(TLS_INSECURE ? { tls: { rejectUnauthorized: false } as any } : {}), + }, + clusterRetryStrategy: () => null, + }); + await cluster.ping(); + console.log(P("cluster ping OK")); + + const nodesInfo: string = (await cluster.call("cluster", ["nodes"])) as string; + let masterAddr = ""; + let replicaAddr = ""; + for (const line of nodesInfo.split("\n")) { + const fields = line.trim().split(/\s+/); + if (fields.length < 3) continue; + const addr = fields[1].split("@")[0]; + if (fields[2].includes("master") && !masterAddr) masterAddr = addr; + if (fields[2].includes("slave") && !replicaAddr) replicaAddr = addr; + } + console.log(P(`master=${masterAddr} replica=${replicaAddr}`)); + + const master = mkClient(masterAddr); + const replica = replicaAddr && replicaAddr !== masterAddr ? mkClient(replicaAddr) : null; + await master.ping(); + if (replica) await replica.ping(); + + // Preload keys + console.log(P(`Preloading ${KEY_COUNT} keys ...`)); + for (let i = 0; i < KEY_COUNT; i++) { + const pipe = master.pipeline(); + pipe.set(k(i), String(i)); + pipe.hset(k(i) + "h", { f0: "v0", f1: "v1" }); + pipe.lpush(k(i) + "l", String(i)); + pipe.rpush(k(i) + "r", String(i)); + pipe.lpush(k(i) + "lt", "a", "b", "c"); + pipe.sadd(k(i) + "s", String(i)); + pipe.sadd(k(i) + "s2", "shared"); + pipe.zadd(k(i) + "z", i % 50, String(i % 50)); + for (let j = i % 5; j < i % 5 + 3; j++) { + pipe.zadd(k(i) + "zr", j, String(j)); + } + await pipe.exec(); + } + console.log(P("Keyspace ready")); + + // Coverage check — standalone + console.log(P("Running coverage check (standalone) ...")); + let covOK = 0, covFail = 0; + const failures: string[] = []; + for (const [name, fn] of SELECTED_CMD_TESTS) { + try { + await fn(master, 0); + covOK++; + } catch (e: any) { + covFail++; + failures.push(`${name}(${e.message?.slice(0, 60)})`); + } + } + console.log(P(` standalone coverage: ${covOK}/${covOK + covFail} passed`)); + for (const f of failures) console.log(P(` FAIL ${f}`)); + + // Coverage check — cluster + console.log(P("Running coverage check (cluster) ...")); + let ccovOK = 0, ccovFail = 0; + const cfailures: string[] = []; + for (const [name, fn] of SELECTED_CMD_TESTS) { + try { + await fn(cluster, 0); + ccovOK++; + } catch (e: any) { + ccovFail++; + cfailures.push(`${name}(${e.message?.slice(0, 60)})`); + } + } + console.log(P(` cluster coverage: ${ccovOK}/${ccovOK + ccovFail} passed`)); + for (const f of cfailures) console.log(P(` FAIL ${f}`)); + + // Stress workers + const nStandalone = WORKERS / 2 | 0; + const nCluster = WORKERS - nStandalone; + const totalStandaloneSlots = nStandalone * INFLIGHT; + const totalClusterSlots = nCluster * INFLIGHT; + console.log(P(`Starting ${nStandalone} standalone + ${nCluster} cluster workers with inflight=${INFLIGHT} (${totalStandaloneSlots} + ${totalClusterSlots} execution slots) ...`)); + const standaloneStats = new CmdStats(); + const clusterStats = new CmdStats(); + const controller = new AbortController(); + + const workers: Promise[] = []; + for (let w = 0; w < totalStandaloneSlots; w++) { + workers.push(stressWorker(master, SELECTED_CMD_TESTS, standaloneStats, controller.signal, w, KEY_COUNT, REPEAT)); + } + for (let w = 0; w < totalClusterSlots; w++) { + workers.push(stressWorker(cluster, SELECTED_CMD_TESTS, clusterStats, controller.signal, w + totalStandaloneSlots, KEY_COUNT, REPEAT)); + } + + const start = Date.now(); + const deadline = DURATION > 0 ? start + DURATION * 1000 : Infinity; + + // Progress reporter + const progressInterval = setInterval(() => { + const elapsed = ((Date.now() - start) / 1000).toFixed(0); + const snap = standaloneStats.snapshot(); + let sOK = 0, sFail = 0; + for (const v of snap.values()) { sOK += v.ok; sFail += v.fail; } + const csnap = clusterStats.snapshot(); + let cOK = 0, cFail = 0; + for (const v of csnap.values()) { cOK += v.ok; cFail += v.fail; } + console.log(P(`progress elapsed=${elapsed}s standalone ok=${sOK} fail=${sFail} cluster ok=${cOK} fail=${cFail}`)); + }, PROGRESS_INTERVAL); + + // Wait for deadline + const checkStop = setInterval(() => { + if (Date.now() >= deadline) { + controller.abort(); + clearInterval(checkStop); + } + }, 100); + + await Promise.all(workers); + clearInterval(progressInterval); + clearInterval(checkStop); + + master.disconnect(); + if (replica) replica.disconnect(); + cluster.disconnect(); + + // Report + function printResult(label: string, s: CmdStats): { totalOK: number; totalFail: number } { + const snap = s.snapshot(); + let totalOK = 0, totalFail = 0; + for (const v of snap.values()) { totalOK += v.ok; totalFail += v.fail; } + console.log(`--- ${label} ---`); + console.log(`total_commands: ok=${totalOK} fail=${totalFail}`); + for (const [name, v] of snap) { + if (v.fail > 0) { + console.log(` ${name}: ok=${v.ok} fail=${v.fail}`); + for (const [signature, count] of topErrorCounts(v.errorTypes)) { + console.log(` error[${count}]: ${signature.slice(0, 200)}`); + } + for (const sample of v.samples) { + console.log(` sample: ${sample.slice(0, 200)}`); + } + } + } + return { totalOK, totalFail }; + } + const sr = printResult("Standalone Client Results", standaloneStats); + const cr = printResult("Cluster Client Results", clusterStats); + const totalOK = sr.totalOK + cr.totalOK; + const totalFail = sr.totalFail + cr.totalFail; + // Tolerate <2% sporadic failures + if (totalFail > 0 && (totalOK === 0 || totalFail / (totalOK + totalFail) > 0.02)) { + process.exit(1); + } +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/tests/e2e/cmd_stress_ts/package-lock.json b/tests/e2e/cmd_stress_ts/package-lock.json new file mode 100644 index 00000000..0e088a0f --- /dev/null +++ b/tests/e2e/cmd_stress_ts/package-lock.json @@ -0,0 +1,679 @@ +{ + "name": "eloqkv-cmd-stress-ts", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "eloqkv-cmd-stress-ts", + "version": "1.0.0", + "dependencies": { + "ioredis": "^5.4.1" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.0.tgz", + "integrity": "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.0.tgz", + "integrity": "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.0.tgz", + "integrity": "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.0.tgz", + "integrity": "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.0.tgz", + "integrity": "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.0.tgz", + "integrity": "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.0.tgz", + "integrity": "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.0.tgz", + "integrity": "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.0.tgz", + "integrity": "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.0.tgz", + "integrity": "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.0.tgz", + "integrity": "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.0.tgz", + "integrity": "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.0.tgz", + "integrity": "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.0.tgz", + "integrity": "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.0.tgz", + "integrity": "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.0.tgz", + "integrity": "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz", + "integrity": "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.0.tgz", + "integrity": "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.0.tgz", + "integrity": "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.0.tgz", + "integrity": "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.0.tgz", + "integrity": "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.0.tgz", + "integrity": "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.0.tgz", + "integrity": "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.0.tgz", + "integrity": "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.0.tgz", + "integrity": "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.0.tgz", + "integrity": "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@ioredis/commands": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", + "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.19.tgz", + "integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esbuild": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz", + "integrity": "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.0", + "@esbuild/android-arm": "0.28.0", + "@esbuild/android-arm64": "0.28.0", + "@esbuild/android-x64": "0.28.0", + "@esbuild/darwin-arm64": "0.28.0", + "@esbuild/darwin-x64": "0.28.0", + "@esbuild/freebsd-arm64": "0.28.0", + "@esbuild/freebsd-x64": "0.28.0", + "@esbuild/linux-arm": "0.28.0", + "@esbuild/linux-arm64": "0.28.0", + "@esbuild/linux-ia32": "0.28.0", + "@esbuild/linux-loong64": "0.28.0", + "@esbuild/linux-mips64el": "0.28.0", + "@esbuild/linux-ppc64": "0.28.0", + "@esbuild/linux-riscv64": "0.28.0", + "@esbuild/linux-s390x": "0.28.0", + "@esbuild/linux-x64": "0.28.0", + "@esbuild/netbsd-arm64": "0.28.0", + "@esbuild/netbsd-x64": "0.28.0", + "@esbuild/openbsd-arm64": "0.28.0", + "@esbuild/openbsd-x64": "0.28.0", + "@esbuild/openharmony-arm64": "0.28.0", + "@esbuild/sunos-x64": "0.28.0", + "@esbuild/win32-arm64": "0.28.0", + "@esbuild/win32-ia32": "0.28.0", + "@esbuild/win32-x64": "0.28.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/ioredis": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", + "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.5.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, + "node_modules/tsx": { + "version": "4.22.3", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.3.tgz", + "integrity": "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/tests/e2e/cmd_stress_ts/package.json b/tests/e2e/cmd_stress_ts/package.json new file mode 100644 index 00000000..21bfc804 --- /dev/null +++ b/tests/e2e/cmd_stress_ts/package.json @@ -0,0 +1,17 @@ +{ + "name": "eloqkv-cmd-stress-ts", + "version": "1.0.0", + "private": true, + "scripts": { + "stress": "npx tsx main.ts", + "stress:prod": "npx tsx main.ts" + }, + "dependencies": { + "ioredis": "^5.4.1" + }, + "devDependencies": { + "tsx": "^4.19.0", + "typescript": "^5.6.0", + "@types/node": "^22.0.0" + } +} diff --git a/tests/e2e/cmd_stress_ts/tsconfig.json b/tests/e2e/cmd_stress_ts/tsconfig.json new file mode 100644 index 00000000..bb9cbd89 --- /dev/null +++ b/tests/e2e/cmd_stress_ts/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["main.ts"] +} diff --git a/tests/e2e/stress.py b/tests/e2e/stress.py deleted file mode 100644 index 4ea9c45e..00000000 --- a/tests/e2e/stress.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -"""Stress test: 30000 concurrent Redis connections via redis-py.""" -import sys, time, threading, argparse - -parser = argparse.ArgumentParser() -parser.add_argument("--host", default="127.0.0.1") -parser.add_argument("--port", type=int, default=6379) -parser.add_argument("--password", default="testpass") -parser.add_argument("--connections", type=int, default=30000) -parser.add_argument("--cmd-timeout", type=int, default=5, help="per-command timeout seconds") -args = parser.parse_args() - -import redis - -lock = threading.Lock() -barrier = threading.Barrier(threading.active_count() + 1) -stop_barrier = None -totals = {"ok": 0, "fail": 0, "ping_ok": 0, "info_ok": 0, "cinfo_ok": 0, "cslots_ok": 0} -times = {"ping": 0.0, "info": 0.0, "cluster_info": 0.0, "cluster_slots": 0.0} -first_errors = {"ping": [], "info": [], "cluster_info": [], "cluster_slots": []} - -def timed_cmd(r, name, fn): - t0 = time.time() - try: - r.execute_command(*fn) if isinstance(fn, tuple) else fn() - finally: - elapsed = time.time() - t0 - with lock: - times[name] += elapsed - -def worker(batch_size): - conns = [] - for _ in range(batch_size): - try: - r = redis.Redis(host=args.host, port=args.port, password=args.password, - socket_timeout=args.cmd_timeout, socket_connect_timeout=args.cmd_timeout, - decode_responses=True) - conns.append(r) - except Exception: - with lock: - totals["fail"] += 1 - barrier.wait() - for r in conns: - cmds = [ - ("ping", lambda: r.ping()), - ("info", lambda: r.info()), - ("cluster_info", ("CLUSTER", "INFO")), - ("cluster_slots", ("CLUSTER", "SLOTS")), - ] - all_ok = True - for cname, fn in cmds: - try: - timed_cmd(r, cname, fn) - except Exception as e: - all_ok = False - with lock: - key = f"{cname}_ok" if cname in ("ping", "info") else \ - f"cinfo_ok" if cname == "cluster_info" else "cslots_ok" - if len(first_errors[cname]) < 3: - first_errors[cname].append(str(e)[:100]) - break - with lock: - if all_ok: - totals["ok"] += 1 - totals["ping_ok"] += 1 - totals["info_ok"] += 1 - totals["cinfo_ok"] += 1 - totals["cslots_ok"] += 1 - else: - totals["fail"] += 1 - stop_barrier.wait() - for r in conns: - try: r.connection_pool.disconnect() - except Exception: pass - -total = args.connections -batch = min(500, total) -thread_count = max(1, total // batch) -barrier = threading.Barrier(thread_count + 1) -stop_barrier = threading.Barrier(thread_count + 1) - -print(f"Connecting {total} clients ({thread_count} threads x {batch} each, timeout={args.cmd_timeout}s)...") -threads = [] -t0 = time.time() -for _ in range(thread_count): - t = threading.Thread(target=worker, args=(batch,)) - t.start() - threads.append(t) -barrier.wait() -t1 = time.time() -print(f"All connected in {t1 - t0:.1f}s. Running commands...") -stop_barrier.wait() -for t in threads: - t.join() -t2 = time.time() - -print(f"--- Results ---") -print(f"connections: total={total} ok={totals['ok']} fail={totals['fail']}") -for key in ("ping_ok", "info_ok", "cinfo_ok", "cslots_ok"): - label = key.replace("_ok", "").replace("cinfo", "CLUSTER INFO").replace("cslots", "CLUSTER SLOTS").replace("ping", "PING").replace("info", "INFO") - print(f" {label}: {totals[key]}/{total} ok") - if first_errors.get(key.replace("_ok", "").replace("cinfo", "cluster_info").replace("cslots", "cluster_slots")): - for e in first_errors[key.replace("_ok", "").replace("cinfo", "cluster_info").replace("cslots", "cluster_slots")][:3]: - print(f" err: {e}") -print(f"timing: total={t2 - t0:.1f}s connect={t1 - t0:.1f}s cmds={t2 - t1:.1f}s") -if totals["fail"] > 0: - print(f"FAIL: {totals['fail']} failures") - sys.exit(1) -print("PASS") diff --git a/tests/e2e/stress_test.sh b/tests/e2e/stress_test.sh deleted file mode 100755 index 5dcf03e5..00000000 --- a/tests/e2e/stress_test.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# Standalone stress test: launch a cluster, run 30K concurrent connections, verify health. -# STEPS=launch,stress,remove # comma-separated steps (default: all) -# CONNECTIONS=30000 # number of concurrent connections -# CMD_TIMEOUT=5 # per-command timeout in seconds -# SKIP_CLEANUP=1 # keep containers after script exits -set -eo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../../" && pwd)" -DOCKER_E2E_DIR="${REPO_ROOT}/tests/docker_ha" -source "${REPO_ROOT}/tests/docker_env.sh" -CLUSTER="test-stress" -TOPO="${SCRIPT_DIR}/topology.generated.yaml" -STEPS="${STEPS:-launch,stress,remove}" -CONNECTIONS="${CONNECTIONS:-30000}" -CMD_TIMEOUT="${CMD_TIMEOUT:-5}" -PASSWD="testpass" -MASTER_HOST="172.28.10.11" -MASTER_PORT="6379" -SSH_KEY="${ELOQCTL_DOCKER_SSH_KEY}" - -cleanup() { - rc=$? - if [ "${KEEP_LOGS:-0}" != "1" ]; then - rm -f "${SCRIPT_DIR}/stress-"*.log "${SCRIPT_DIR}/launch-stress.log" "${TOPO}" - fi - exit "${rc}" -} -trap cleanup EXIT - -step() { - local name="$1"; shift - if [[ ",${STEPS}," == *",${name},"* ]]; then - "$@" - else - echo "[skip] ${name}" - fi -} - -do_launch() { - echo "=== Launch cluster ===" - render_topology "${SCRIPT_DIR}/topology.yaml" "${TOPO}" - start_docker_env - "${ELOQCTL}" stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true - "${ELOQCTL}" remove "${CLUSTER}" --force >/dev/null 2>&1 || true - run_with_progress 300 "${SCRIPT_DIR}/launch-stress.log" "${ELOQCTL}" launch "${TOPO}" || { - echo "FAIL: launch failed"; dump_failure_diagnostics "${SCRIPT_DIR}/launch-stress.log"; exit 1 - } - run_with_progress 180 "${SCRIPT_DIR}/launch-stress.log" "${ELOQCTL}" status "${CLUSTER}" --wait 120 >/dev/null 2>&1 || { - echo "FAIL: cluster not healthy after launch"; exit 1 - } - echo " cluster ready" -} - -do_stress() { - echo "=== Stress test: ${CONNECTIONS} connections ===" - echo " installing redis-py..." - for port in 2221 2222 2223; do - ssh_cmd "${port}" "sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-pip >/dev/null 2>&1 && pip3 install --quiet redis >/dev/null 2>&1" & - done - wait - scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - -o PasswordAuthentication=no -o BatchMode=yes -o ConnectTimeout=10 \ - -i "${SSH_KEY}" -P 2221 \ - "${SCRIPT_DIR}/stress.py" "eloq@127.0.0.1:/home/eloq/${CLUSTER}/stress.py" \ - >/dev/null 2>&1 || { echo "FAIL: cannot upload stress script"; exit 1; } - ssh_cmd 2221 "python3 /home/eloq/${CLUSTER}/stress.py --host ${MASTER_HOST} --port ${MASTER_PORT} --password ${PASSWD} --connections ${CONNECTIONS} --cmd-timeout ${CMD_TIMEOUT}" 2>&1 || { - echo "FAIL: stress test failed"; exit 1 - } - echo " verifying cluster health..." - "${ELOQCTL}" status "${CLUSTER}" --wait 30 >/dev/null 2>&1 || { echo "FAIL: unhealthy after stress"; exit 1; } - echo " OK" -} - -do_remove() { - echo "=== Remove cluster ===" - "${ELOQCTL}" stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true - "${ELOQCTL}" remove "${CLUSTER}" --force >/dev/null 2>&1 || true - compose_down - echo " removed" -} - -step launch do_launch -step stress do_stress -step remove do_remove - -echo "PASS" diff --git a/tests/e2e/test.sh b/tests/e2e/test.sh deleted file mode 100755 index ad6bf00b..00000000 --- a/tests/e2e/test.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/bin/bash -# End-to-end test: deploy EloqKV once, run all scenarios sequentially. -# SKIP_LAUNCH=1 - skip cluster launch (cluster must already be running) -# SKIP_CLEANUP=1 - skip stop/remove cleanup -set -eo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../../" && pwd)" -DOCKER_E2E_DIR="${REPO_ROOT}/tests/docker_ha" -source "${REPO_ROOT}/tests/docker_env.sh" -CLUSTER="test-e2e" -TOPO="${SCRIPT_DIR}/topology.generated.yaml" -LAUNCH_TIMEOUT_SECONDS="${LAUNCH_TIMEOUT_SECONDS:-120}" -STATUS_TIMEOUT_SECONDS="${STATUS_TIMEOUT_SECONDS:-120}" -SKIP_LAUNCH="${SKIP_LAUNCH:-0}" -SKIP_CLEANUP="${SKIP_CLEANUP:-0}" - -cleanup() { - rc=$? - if [ "${SKIP_CLEANUP}" != "1" ]; then - timeout --kill-after=5s "${CLEANUP_TIMEOUT_SECONDS}s" "${ELOQCTL}" stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true - timeout --kill-after=5s "${CLEANUP_TIMEOUT_SECONDS}s" "${ELOQCTL}" remove "${CLUSTER}" --force >/dev/null 2>&1 || true - fi - compose_down - if [ "${KEEP_E2E_LOGS:-0}" != "1" ]; then - rm -f "${SCRIPT_DIR}/"*.log "${TOPO}" "${SCRIPT_DIR}/exported.yaml" - fi - if [ ${rc} -ne 0 ]; then - echo "FAIL: e2e suite failed" - fi - exit "${rc}" -} -trap cleanup EXIT - -render_topology "${SCRIPT_DIR}/topology.yaml" "${TOPO}" -start_docker_env - -# ---- [1] Launch cluster ---- -if [ "${SKIP_LAUNCH}" != "1" ]; then -echo "[1/6] Launch cluster" -"${ELOQCTL}" stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true -"${ELOQCTL}" remove "${CLUSTER}" --force >/dev/null 2>&1 || true -set +e -run_with_progress "${LAUNCH_TIMEOUT_SECONDS}" "${SCRIPT_DIR}/launch.log" "${ELOQCTL}" launch "${TOPO}" -launch_rc=$? -set -e -if [ ${launch_rc} -ne 0 ]; then - echo "FAIL: launch exited ${launch_rc}" - dump_failure_diagnostics "${SCRIPT_DIR}/launch.log" - exit 1 -fi -echo " OK" -fi - -# ---- [2] Verify status ---- -echo "[2/6] Verify cluster status" -run_with_progress "${STATUS_TIMEOUT_SECONDS}" "${SCRIPT_DIR}/status.log" "${ELOQCTL}" status "${CLUSTER}" --wait 90 || { - echo "FAIL: status --wait failed" - dump_failure_diagnostics "${SCRIPT_DIR}/status.log" - exit 1 -} -echo " OK" - -# ---- [3] Read-only commands ---- -echo "[3/6] Test read-only commands" -"${ELOQCTL}" versions > "${SCRIPT_DIR}/versions.log" 2>&1 && \ - grep -q "1." "${SCRIPT_DIR}/versions.log" || echo " versions N/A (PG unreachable)" - -"${ELOQCTL}" list > "${SCRIPT_DIR}/list.log" 2>&1 || { echo "FAIL: list"; exit 1; } -grep -q "${CLUSTER}" "${SCRIPT_DIR}/list.log" || { echo "FAIL: list"; exit 1; } - -"${ELOQCTL}" export "${CLUSTER}" --output "${SCRIPT_DIR}/exported.yaml" > "${SCRIPT_DIR}/export.log" 2>&1 || { echo "FAIL: export"; exit 1; } -grep -q "cluster_name" "${SCRIPT_DIR}/exported.yaml" || { echo "FAIL: export"; exit 1; } - -CLIENT=$("${ELOQCTL}" -q connect "${CLUSTER}" 2>"${SCRIPT_DIR}/connect.log") -echo " OK" - -# ---- [4] Rolling update via apply ---- -echo "[4/6] Rolling update (apply checkpoint_interval change)" -"${ELOQCTL}" status "${CLUSTER}" >/dev/null 2>&1 -# Generate a modified topology with different checkpoint_interval from the rendered version -sed 's/checkpoint_interval: 120/checkpoint_interval: 130/' "${TOPO}" > "${SCRIPT_DIR}/topology-v2.yaml" -"${ELOQCTL}" plan "${SCRIPT_DIR}/topology-v2.yaml" > "${SCRIPT_DIR}/plan.log" 2>&1 || { echo "FAIL: plan"; cat "${SCRIPT_DIR}/plan.log"; exit 1; } -"${ELOQCTL}" apply "${SCRIPT_DIR}/topology-v2.yaml" > "${SCRIPT_DIR}/apply.log" 2>&1 || { echo "FAIL: apply"; exit 1; } -run_with_progress "${STATUS_TIMEOUT_SECONDS}" "${SCRIPT_DIR}/post-apply.log" "${ELOQCTL}" status "${CLUSTER}" --wait 60 || { - echo "FAIL: status after apply" - dump_failure_diagnostics "${SCRIPT_DIR}/post-apply.log" - exit 1 -} -rm -f "${SCRIPT_DIR}/topology-v2.yaml" -echo " OK" - -# ---- [5] Scale add + remove ---- -echo "[5/6] Scale add standby" -"${ELOQCTL}" status "${CLUSTER}" >/dev/null 2>&1 -"${ELOQCTL}" scale "${CLUSTER}" --add-nodes "172.28.10.11:6390" --ng-id 0 --is-candidate true > "${SCRIPT_DIR}/scale-add.log" 2>&1 || { - echo "FAIL: scale add" - dump_failure_diagnostics "${SCRIPT_DIR}/scale-add.log" - exit 1 -} -run_with_progress "${STATUS_TIMEOUT_SECONDS}" "${SCRIPT_DIR}/post-scale-add.log" "${ELOQCTL}" status "${CLUSTER}" --wait 60 || { - echo "FAIL: status after scale add" - exit 1 -} -echo " OK" - -echo "[6/6] Scale remove old standby" -"${ELOQCTL}" scale "${CLUSTER}" --remove-nodes "172.28.10.12:6379" > "${SCRIPT_DIR}/scale-remove.log" 2>&1 || { - echo "FAIL: scale remove" - dump_failure_diagnostics "${SCRIPT_DIR}/scale-remove.log" - exit 1 -} -run_with_progress "${STATUS_TIMEOUT_SECONDS}" "${SCRIPT_DIR}/post-scale-rm.log" "${ELOQCTL}" status "${CLUSTER}" --wait 60 || { - echo "FAIL: status after scale remove" - exit 1 -} -echo " OK" - -# ---- [7] Stop + start cycle ---- -echo "[7/14] Stop cluster and validate topology" -"${ELOQCTL}" stop "${CLUSTER}" --all --force > "${SCRIPT_DIR}/stop.log" 2>&1 || { echo "FAIL: stop"; exit 1; } -sleep 3 -"${ELOQCTL}" check "${TOPO}" > "${SCRIPT_DIR}/check.log" 2>&1 && echo " OK" || { echo "FAIL: check valid topology"; cat "${SCRIPT_DIR}/check.log"; exit 1; } - -echo "[8/14] Restart cluster" -"${ELOQCTL}" start "${CLUSTER}" > "${SCRIPT_DIR}/start.log" 2>&1 || { echo "FAIL: start"; exit 1; } -run_with_progress "${STATUS_TIMEOUT_SECONDS}" "${SCRIPT_DIR}/post-start.log" "${ELOQCTL}" status "${CLUSTER}" --wait 60 || { - echo "FAIL: status after restart" - exit 1 -} -echo " OK" - -# ---- [9] Failover ---- -echo "[9/14] Failover standby to master" -"${ELOQCTL}" failover "${CLUSTER}" \ - --old-leader-host 172.28.10.11 --old-leader-port 6379 \ - --new-leader-host 172.28.10.11 --new-leader-port 6390 \ - > "${SCRIPT_DIR}/failover.log" 2>&1 || { - echo "FAIL: failover" - cat "${SCRIPT_DIR}/failover.log" - exit 1 -} -run_with_progress "${STATUS_TIMEOUT_SECONDS}" "${SCRIPT_DIR}/post-failover.log" "${ELOQCTL}" status "${CLUSTER}" --wait 60 || { - echo "FAIL: status after failover" - exit 1 -} -echo " OK" - -# ---- [10] Monitor + log-service status ---- -echo "[10/14] Monitor and log-service status" -"${ELOQCTL}" monitor status "${CLUSTER}" > "${SCRIPT_DIR}/monitor-status.log" 2>&1 && echo " monitor OK" || echo " monitor N/A (skipped)" -"${ELOQCTL}" log-service status "${CLUSTER}" > "${SCRIPT_DIR}/log-status.log" 2>&1 && echo " log-srv OK" || echo " log-srv N/A (skipped)" -echo " OK" - -# ---- [11] Exec custom command ---- -echo "[11/14] Exec custom remote command" -"${ELOQCTL}" exec "uname -a" "${TOPO}" > "${SCRIPT_DIR}/exec.log" 2>&1 || { echo "FAIL: exec"; cat "${SCRIPT_DIR}/exec.log"; exit 1; } -echo " OK" - -# ---- [12] Schema upgrade ---- -echo "[12/14] Schema upgrade" -"${ELOQCTL}" upgrade > "${SCRIPT_DIR}/upgrade.log" 2>&1 || { echo "FAIL: upgrade"; exit 1; } -echo " OK" - -# ---- [11] Remove cluster ---- -echo "[13/14] Remove cluster" -"${ELOQCTL}" stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true -"${ELOQCTL}" remove "${CLUSTER}" --force > "${SCRIPT_DIR}/remove.log" 2>&1 || { echo "FAIL: remove"; exit 1; } -"${ELOQCTL}" list > "${SCRIPT_DIR}/post-remove-list.log" 2>&1 -grep -q "${CLUSTER}" "${SCRIPT_DIR}/post-remove-list.log" && { echo "FAIL: cluster still present after remove"; exit 1; } -echo " OK" - -echo "" -echo "PASS: all E2E tests completed" diff --git a/tests/e2e/topology.yaml b/tests/e2e/topology.yaml index 118707e2..3986c25c 100644 --- a/tests/e2e/topology.yaml +++ b/tests/e2e/topology.yaml @@ -3,63 +3,40 @@ connection: auth_type: "keypair" auth: keypair: "${ELOQCTL_DOCKER_SSH_KEY}" - ssh_endpoints: - 172.28.10.11: - host: "127.0.0.1" - port: 2221 - 172.28.10.12: - host: "127.0.0.1" - port: 2222 - 172.28.10.13: - host: "127.0.0.1" - port: 2223 - service_endpoints: - 172.28.10.11:6379: - host: "127.0.0.1" - port: 16371 - 172.28.10.11:16380: - host: "127.0.0.1" - port: 26371 - 172.28.10.11:6390: - host: "127.0.0.1" - port: 16391 - 172.28.10.12:6379: - host: "127.0.0.1" - port: 16372 - 172.28.10.12:16380: - host: "127.0.0.1" - port: 26372 - 172.28.10.13:6379: - host: "127.0.0.1" - port: 16373 - 172.28.10.13:16380: - host: "127.0.0.1" - port: 26373 deployment: cluster_name: "test-e2e" product: "EloqKV" - version: "1.2.2" + version: "${ELOQKV_VERSION}" install_dir: "/home/eloq" enable_wal: false enable_io_uring: false + enable_tls: true cluster_mode: true - maxclients: 60000 checkpoint_interval: 120 hardware: 172.28.10.11: - cpu: 2 + cpu: 7 memory: 1024 + event_dispatcher_num: 1 172.28.10.12: - cpu: 2 + cpu: 7 memory: 1024 + event_dispatcher_num: 1 172.28.10.13: cpu: 2 memory: 1024 + 172.28.10.14: + cpu: 4 + memory: 1024 tx_service: tx_host_ports: [172.28.10.11:6379] standby_host_ports: [172.28.10.12:6379] voter_host_ports: [172.28.10.13:6379] requirepass: "testpass" storage_service: - rocksdb: !LOCAL - path: "/home/eloq/rocksdb" + rocksdb: !MINIO + aws_access_key_id: "minioadmin" + aws_secret_key: "minioadmin" + bucket_name: "eloqservice" + bucket_prefix: "store" + endpoint: "http://172.28.10.20:9000" diff --git a/tests/install_control_eloqctl.sh b/tests/install_control_eloqctl.sh new file mode 100755 index 00000000..d6fd869a --- /dev/null +++ b/tests/install_control_eloqctl.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="${REPO_ROOT:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +DOCKER_E2E_DIR="${DOCKER_E2E_DIR:-${REPO_ROOT}/tests/docker_ha}" + +source "${REPO_ROOT}/tests/docker_env.sh" + +BINARY_PATH="${1:-${REPO_ROOT}/target/debug/cluster_mgr}" +CONFIG_DIR="${REPO_ROOT}/src/cluster_mgr/config" +SSH_KEY_PATH="${DOCKER_E2E_DIR}/id_ed25519" +INSTALL_ROOT="${CONTROL_ELOQCTL_HOME}" +INSTALL_BIN="${INSTALL_ROOT}/bin/cluster_mgr" + +if [ ! -x "${BINARY_PATH}" ]; then + echo "missing executable binary: ${BINARY_PATH}" >&2 + echo "build it first with: cargo build -p cluster_mgr --bin cluster_mgr" >&2 + exit 1 +fi + +if [ ! -d "${CONFIG_DIR}" ]; then + echo "missing config directory: ${CONFIG_DIR}" >&2 + exit 1 +fi + +if [ ! -f "${SSH_KEY_PATH}" ]; then + echo "missing ssh key: ${SSH_KEY_PATH}" >&2 + exit 1 +fi + +compose ps "${CONTROL_NODE_SERVICE}" >/dev/null + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +cp "${BINARY_PATH}" "${TMP_DIR}/cluster_mgr" +cp -R "${CONFIG_DIR}" "${TMP_DIR}/config" +cp "${SSH_KEY_PATH}" "${TMP_DIR}/id_ed25519" + +compose exec -T "${CONTROL_NODE_SERVICE}" bash -lc " + set -euo pipefail + install -d -m 700 -o eloq -g eloq /home/eloq/.ssh + install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}' + install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}/bin' + rm -rf '${INSTALL_ROOT}/config' +" + +compose cp "${TMP_DIR}/cluster_mgr" "${CONTROL_NODE_SERVICE}:${INSTALL_BIN}" +compose cp "${TMP_DIR}/config" "${CONTROL_NODE_SERVICE}:${INSTALL_ROOT}/config" +compose cp "${TMP_DIR}/id_ed25519" "${CONTROL_NODE_SERVICE}:${ELOQCTL_CONTROL_SSH_KEY}" + +compose exec -T "${CONTROL_NODE_SERVICE}" bash -lc " + set -euo pipefail + chown -R eloq:eloq '${INSTALL_ROOT}' /home/eloq/.ssh + chmod 755 '${INSTALL_BIN}' + chmod 600 '${ELOQCTL_CONTROL_SSH_KEY}' + printf '%s\n' '#!/bin/bash' \ + 'export HOME=/home/eloq' \ + 'export ELOQCTL_HOME=/home/eloq/.eloqctl' \ + 'exec /home/eloq/.eloqctl/bin/cluster_mgr "\$@"' \ + > /usr/local/bin/eloqctl + chmod 755 /usr/local/bin/eloqctl +" + +echo "installed eloqctl into ${CONTROL_NODE_SERVICE}" +echo "login with: ssh -i \"${SSH_KEY_PATH}\" -p 2224 eloq@127.0.0.1" +echo "run with: eloqctl --help"