diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af1f2890..1e1b1f47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,15 +16,15 @@ on: workers: description: "Stress test workers" required: false - default: "32" + default: "8" duration: description: "Stress duration (seconds)" required: false - default: "60" + default: "20" repeat: description: "Per-command repeat factor" required: false - default: "10" + default: "8" inflight: description: "Concurrent execution slots per worker" required: false @@ -74,29 +74,19 @@ jobs: - name: Build eloqctl run: | - cargo build -p cluster_mgr - mkdir -p ~/.eloqctl/bin - cp target/debug/cluster_mgr ~/.eloqctl/bin/cluster_mgr - chmod 755 ~/.eloqctl/bin/cluster_mgr - rm -f ~/.eloqctl/config - ln -s "$PWD/src/cluster_mgr/config" ~/.eloqctl/config + tests/e2e/devctl.sh build - name: Run E2E test run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - workers="${{ github.event.inputs.workers || '32' }}" - duration="${{ github.event.inputs.duration || '60' }}" - repeat="${{ github.event.inputs.repeat || '10' }}" + workers="${{ github.event.inputs.workers || '8' }}" + duration="${{ github.event.inputs.duration || '20' }}" + repeat="${{ github.event.inputs.repeat || '8' }}" inflight="${{ github.event.inputs.inflight || '4' }}" - elif [ "${{ github.event_name }}" = "pull_request" ]; then - workers="16" - duration="60" - repeat="10" - inflight="4" else - workers="32" - duration="60" - repeat="20" + workers="8" + duration="20" + repeat="8" inflight="4" fi @@ -105,13 +95,14 @@ jobs: export ELOQCTL="$HOME/.eloqctl/bin/cluster_mgr" export ELOQKV_VERSION="${{ github.event.inputs.version || '1.2.2' }}" export DURATION_SECONDS="${duration}" + export INFO_ONLY_DURATION_SECONDS="30" export WORKERS="${workers}" export INFLIGHT="${inflight}" export REPEAT="${repeat}" export SKIP_DEPS=1 - export STEPS="launch,py-stress,go-stress,ts-stress,remove" + export STEPS="launch,monitor-update,eloqctl-mutate,py-stress,go-stress,ts-stress,remove" echo "Running E2E with WORKERS=${WORKERS} INFLIGHT=${INFLIGHT} DURATION_SECONDS=${DURATION_SECONDS} REPEAT=${REPEAT}" - bash tests/e2e/cmd_stress_test.sh + tests/e2e/devctl.sh stress "${STEPS}" - name: Cleanup Docker if: always() diff --git a/Cargo.lock b/Cargo.lock index 5003cf64..071bae69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -347,6 +347,15 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "async-trait" version = "0.1.81" @@ -931,9 +940,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytecount" @@ -1086,7 +1095,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cluster_mgr" -version = "1.6.6" +version = "1.6.7" dependencies = [ "anyhow", "async-trait", @@ -1132,7 +1141,7 @@ dependencies = [ "sysinfo 0.30.12", "tabled", "tar", - "thiserror", + "thiserror 1.0.61", "tokio", "tokio-postgres", "toml 0.8.14", @@ -1145,6 +1154,7 @@ dependencies = [ "uuid", "walkdir", "whoami", + "zip", ] [[package]] @@ -1344,9 +1354,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1517,6 +1527,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "derive_more" version = "0.99.18" @@ -1532,7 +1553,7 @@ dependencies = [ [[package]] name = "devtools" -version = "1.6.6" +version = "1.6.7" dependencies = [ "anyhow", "async-trait", @@ -1549,7 +1570,7 @@ dependencies = [ "serde", "serde_derive", "sysinfo 0.27.8", - "thiserror", + "thiserror 1.0.61", "tokio", "toml 0.5.11", ] @@ -1621,6 +1642,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "dlv-list" version = "0.5.2" @@ -2616,7 +2648,7 @@ checksum = "612ed4ea9ce5acfb5d26339302528a5e1e59dfed95e9e11af3c083236ff1d15d" dependencies = [ "libc", "neli", - "thiserror", + "thiserror 1.0.61", "windows-sys 0.48.0", ] @@ -2638,9 +2670,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" [[package]] name = "lru" @@ -3570,6 +3602,7 @@ dependencies = [ "futures-util", "itoa", "log", + "native-tls", "num-bigint", "percent-encoding", "pin-project-lite", @@ -3578,6 +3611,7 @@ dependencies = [ "sha1_smol", "socket2 0.6.3", "tokio", + "tokio-native-tls", "tokio-util", "url", ] @@ -3608,7 +3642,7 @@ checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ "getrandom 0.2.15", "libredox", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -3705,7 +3739,7 @@ dependencies = [ [[package]] name = "rest_api" -version = "1.6.6" +version = "1.6.7" dependencies = [ "actix-server", "actix-web", @@ -3809,7 +3843,7 @@ dependencies = [ "sha1", "sha2", "subtle", - "thiserror", + "thiserror 1.0.61", "tokio", "tokio-util", ] @@ -3823,7 +3857,7 @@ dependencies = [ "dirs-next", "futures", "log", - "thiserror", + "thiserror 1.0.61", "tokio", "whoami", ] @@ -3872,7 +3906,7 @@ dependencies = [ "serde", "sha1", "sha2", - "thiserror", + "thiserror 1.0.61", "tokio", "tokio-stream", "yasna", @@ -4246,6 +4280,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "siphasher" version = "0.3.11" @@ -4373,7 +4413,7 @@ dependencies = [ "sha2", "smallvec", "sqlformat", - "thiserror", + "thiserror 1.0.61", "tokio", "tokio-stream", "tracing", @@ -4457,7 +4497,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 1.0.61", "tracing", "whoami", ] @@ -4496,7 +4536,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 1.0.61", "tracing", "whoami", ] @@ -4710,7 +4750,16 @@ version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.61", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -4724,6 +4773,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -5666,6 +5726,35 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.2.6", + "memchr", + "thiserror 2.0.18", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/README.md b/README.md index 02812f60..29c12420 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ The active cluster manager product target is **EloqKV**. Legacy EloqSQL, MonoSQL Install the latest release: ```sh -curl -fsSL https://raw.githubusercontent.com/eloqdb/eloq_waiter/main/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/monographdb/eloq_waiter/main/install.sh | sh ``` Install a specific release tag: ```sh -curl -fsSL https://raw.githubusercontent.com/eloqdb/eloq_waiter/main/install.sh | sh -s -- v1.4.0 +curl -fsSL https://raw.githubusercontent.com/monographdb/eloq_waiter/main/install.sh | sh -s -- v1.6.7 ``` For local development, build and install the current checkout: @@ -26,6 +26,9 @@ scripts/install-dev.sh This installs `${ELOQCTL_HOME:-$HOME/.eloqctl}/bin/cluster_mgr` and links `${ELOQCTL_HOME:-$HOME/.eloqctl}/config` to `src/cluster_mgr/config` in this repository. +When the installer detects an existing `eloqctl` state directory, it also runs +`eloqctl upgrade` automatically to migrate local cluster metadata. + ## Quick Start Launch an EloqKV cluster from a topology file: diff --git a/doc/cluster_mgr.md b/doc/cluster_mgr.md index 8b6f4b5c..748214de 100644 --- a/doc/cluster_mgr.md +++ b/doc/cluster_mgr.md @@ -49,12 +49,13 @@ eloqctl remove eloqkv-cluster --force | `connect ` | Print an EloqKV client command for the cluster. | | `plan ` | Preview supported changes without mutating local state or remote hosts. | | `apply ` | Execute the same plan shown by `plan`, gated by live critical-service health and verified afterward. | -| `update ` | Rolling version update for an existing cluster. Version lookup and tarball selection come from GitHub Releases for `eloqdata/eloqkv`. `--download-only` resolves and caches the required tarballs without touching remote hosts. `--monitor` updates only one monitor component. | +| `update ` | Rolling version update for an existing cluster. Version lookup and tarball selection come from GitHub Releases for `eloqdata/eloqkv`. `--download-only` resolves and caches the required tarballs without touching remote hosts. | | `update-conf ` | Apply selected config fields and optionally restart tx nodes. | | `scale ` | Add or remove EloqKV tx/standby nodes. Duplicate add/remove requests are no-ops. | | `scalelog ` | Add or remove log service nodes. Duplicate add/remove requests are no-ops. | | `failover ` | Move leadership from an old leader to a requested new leader. | -| `monitor start|stop ` | Manage Prometheus, Grafana, and node exporter when monitor config is present. | +| `monitor start|stop|status ` | Manage Prometheus, Grafana, and node exporter when monitor config is present. | +| `monitor update --component [--url ]` | Update exactly one monitor component without touching EloqKV. If the selected component is not running yet, the command installs/unpacks it and then starts it. | | `log-service start|stop ` | Manage standalone log service nodes. | | `backup ...` | Create, list, remove, restore, and dump backups. | | `export [--output file]` | Write the saved launch-compatible topology YAML. | diff --git a/doc/upgrade.md b/doc/upgrade.md index 2ef466b0..0068f7ea 100644 --- a/doc/upgrade.md +++ b/doc/upgrade.md @@ -5,7 +5,7 @@ Install the desired release tag with the installer: ```sh -curl -fsSL https://raw.githubusercontent.com/eloqdb/eloq_waiter/main/install.sh | sh -s -- vX.Y.Z +curl -fsSL https://raw.githubusercontent.com/monographdb/eloq_waiter/main/install.sh | sh -s -- v1.6.7 ``` For local development builds, reinstall from the current checkout: @@ -14,6 +14,10 @@ For local development builds, reinstall from the current checkout: scripts/install-dev.sh ``` +The release installer now runs `eloqctl upgrade` automatically when it detects +an existing local state directory, so previously registered clusters remain +visible after an upgrade. + ## Upgrade Local State Schema Run the SQLite schema upgrade command after installing a newer `eloqctl` if local state needs migration: @@ -53,15 +57,17 @@ eloqctl update 1.2.2 Update only one monitor component without touching the EloqKV cluster itself: ```sh -eloqctl update --monitor grafana +eloqctl monitor update --cluster --component grafana ``` Override the monitor tarball URL for a monitor-only update: ```sh -eloqctl update --monitor grafana --monitor-url https://dl.grafana.com/oss/release/grafana-11.0.0.linux-amd64.tar.gz +eloqctl monitor update --cluster --component grafana --url https://dl.grafana.com/oss/release/grafana-11.0.0.linux-amd64.tar.gz ``` +If the selected monitor component is configured in topology but not currently deployed or running, `eloqctl monitor update` still downloads, unpacks, uploads config, and starts it. + GitHub-hosted EloqKV tarballs are now validated with the asset `sha256` digest when available. Cached files are reused only when the digest still matches, so a re-published asset under the same tag can be refreshed correctly. Upgrade to the latest available version: diff --git a/install.sh b/install.sh index b84b4265..10ef39d6 100644 --- a/install.sh +++ b/install.sh @@ -36,9 +36,16 @@ if [ -z "${ELOQCTL_HOME:-}" ]; then fi BIN_DIR="${ELOQCTL_HOME}/bin" +BIN_PATH="${BIN_DIR}/cluster_mgr" +STATE_DB_PATH="${ELOQCTL_HOME}/db/cluster_mgr_state.db" TMP_TARBALL="${TMPDIR:-/tmp}/eloqctl.tar.gz" mkdir -p "${BIN_DIR}" +HAD_EXISTING_INSTALL=false +if [ -x "${BIN_PATH}" ] || [ -f "${STATE_DB_PATH}" ]; then + HAD_EXISTING_INSTALL=true +fi + resolve_latest_tag() { latest_tag="$( curl -fsSL \ @@ -94,6 +101,15 @@ fi chmod 755 "${BIN_DIR}/cluster_mgr" +if [ "${HAD_EXISTING_INSTALL}" = true ]; then + echo "Running local state upgrade..." + if ! ELOQCTL_HOME="${ELOQCTL_HOME}" "${BIN_PATH}" upgrade; then + echo "WARNING: eloqctl was installed, but local state upgrade failed." >&2 + echo "Run this manually before using existing clusters:" >&2 + echo " ELOQCTL_HOME=${ELOQCTL_HOME} ${BIN_PATH} upgrade" >&2 + fi +fi + print_completion_help() { echo "Shell completion is available but is not enabled automatically." case "${shell}" in diff --git a/src/cluster_mgr/Cargo.toml b/src/cluster_mgr/Cargo.toml index 23d2f047..3766d812 100644 --- a/src/cluster_mgr/Cargo.toml +++ b/src/cluster_mgr/Cargo.toml @@ -39,6 +39,7 @@ serde_yaml = "0.9" serde_with = "1.6.1" sha2 = "0.10" tar = "0.4" +zip = { version = "2", default-features = false, features = ["deflate"] } sqlx = { version = "0.7.3", features = [ "runtime-tokio-native-tls", "sqlite", @@ -81,7 +82,7 @@ users = "0.11.0" regex = "1.10.2" toml = "0.8.10" tokio-postgres = "0.7.10" -redis = { version = "0.32.7", features = ["cluster", "cluster-async", "aio", "tokio-comp"] } +redis = { version = "0.32.7", features = ["cluster", "cluster-async", "aio", "tokio-native-tls-comp"] } uuid = { version = "1", features = ["v4"] } prettytable = "0.10.0" aws-sdk-s3 = "1.0" diff --git a/src/cluster_mgr/config/alert.rules b/src/cluster_mgr/config/alert.rules index ca71deed..d264ce11 100644 --- a/src/cluster_mgr/config/alert.rules +++ b/src/cluster_mgr/config/alert.rules @@ -115,14 +115,14 @@ groups: summary: "Leader changes frequently" description: "More than three leadership changes were seen within 10 minutes for node group {{ $labels.ng_id }}" - - alert: eloqkv_instance_ops_too_low - expr: sum by (instance) (rate(redis_command_aggregated_total[1m])) < 100000 + - alert: eloqkv_cluster_ops_too_low + expr: sum(rate(redis_command_aggregated_total[1m])) < 100000 for: 1m labels: severity: critical annotations: - summary: "EloqKV instance OPS too low" - description: "EloqKV instance {{ $labels.instance }} OPS has been below 10000 for 1 minutes. Current value: {{ printf \"%.2f\" $value }}" + summary: "EloqKV cluster OPS too low" + description: "EloqKV cluster ${CLUSTER_NAME} OPS has been below 100000 for 1 minute. Current value: {{ printf \"%.2f\" $value }}" - alert: eloqkv_active_connections_too_high expr: sum by (instance) (redis_connection_count) > 100 diff --git a/src/cluster_mgr/config/feishu.zh.tmpl b/src/cluster_mgr/config/feishu.zh.tmpl new file mode 100644 index 00000000..29bfe406 --- /dev/null +++ b/src/cluster_mgr/config/feishu.zh.tmpl @@ -0,0 +1,63 @@ +{{ define "severity.tag" -}} +{{- if eq . "critical" -}}严重 +{{- else if eq . "warning" -}}警告 +{{- else if eq . "info" -}}信息 +{{- else -}}未知 +{{- end -}} +{{- end }} + +{{ define "status.tag" -}} +{{- if eq . "firing" -}}告警中 +{{- else -}}已恢复 +{{- end -}} +{{- end }} + +{{ define "prom.title" -}} +{{ if eq .Status "firing" }}[告警中]{{ else }}[已恢复]{{ end }} {{ if .CommonLabels.severity }}{{ .CommonLabels.severity }}{{ else }}unknown{{ end }} · {{ if .CommonLabels.alertname }}{{ .CommonLabels.alertname }}{{ else }}EloqKV 监控告警{{ end }} · {{ len .Alerts }} +{{- end }} + +{{ define "prom.text" -}} +{{ if .Signature }}{{ .Signature }}{{ else }}EloqKV{{ end }} 告警通知 +状态: {{ if eq .Status "firing" }}告警中{{ else }}已恢复{{ end }} +级别: {{ if .CommonLabels.severity }}{{ .CommonLabels.severity }}{{ else }}未知{{ end }} +规则: {{ if .CommonLabels.alertname }}{{ .CommonLabels.alertname }}{{ else }}EloqKV 监控告警{{ end }} +告警数量: {{ len .Alerts }} +通知时间: {{ .MessageAt.Format "2006-01-02 15:04:05 MST" }} +{{ if .CommonAnnotations.summary }}摘要: {{ .CommonAnnotations.summary }} +{{ else if .CommonAnnotations.description }}摘要: {{ .CommonAnnotations.description }} +{{ end }} +{{ range .Alerts -}} + +实例: {{ if .Labels.instance }}{{ .Labels.instance }}{{ else }}未知{{ end }} +状态: {{ if eq .Status "firing" }}告警中{{ else }}已恢复{{ end }} +开始时间: {{ .StartsAt.Format "2006-01-02 15:04:05 MST" }} +{{ if .Annotations.summary }}摘要: {{ .Annotations.summary }} +{{ end }}{{ if .Annotations.description }}描述: {{ .Annotations.description }} +{{ end -}} +{{ end }} +{{ if .ExternalURL }}Alertmanager: {{ .ExternalURL }}{{ end }} +{{- end }} + +{{ define "prom.markdown" -}} +{{ if .Signature }}{{ .Signature }}{{ else }}EloqKV{{ end }} **告警通知** +{{ template "status.tag" .Status }} {{ if .CommonLabels.severity }}{{ template "severity.tag" .CommonLabels.severity }}{{ else }}未知级别{{ end }} **{{ if .CommonLabels.alertname }}{{ .CommonLabels.alertname }}{{ else }}EloqKV 监控告警{{ end }}** + +{{ with index .Alerts 0 -}} +**首个实例**：{{ if .Labels.instance }}{{ .Labels.instance }}{{ else }}未知{{ end }} +{{- end }} +**影响实例**：{{ len .Alerts }} +**通知时间**：{{ .MessageAt.Format "2006-01-02 15:04:05 MST" }} + +**摘要**：{{ if .CommonAnnotations.summary }}{{ .CommonAnnotations.summary }}{{ else if .CommonAnnotations.description }}{{ .CommonAnnotations.description }}{{ else }}请查看下方实例详情。{{ end }} + +**实例详情** +{{ range .Alerts -}} +- {{ if .Labels.instance }}{{ .Labels.instance }}{{ else }}未知{{ end }} · {{ template "status.tag" .Status }} · {{ .StartsAt.Format "2006-01-02 15:04:05 MST" }} +{{ if .Annotations.summary }} 摘要：{{ .Annotations.summary }} +{{ end }}{{ if .Annotations.description }} 描述：{{ .Annotations.description }} +{{ end -}} +{{ end }} +{{ if .ExternalURL }} +[查看 Alertmanager]({{ .ExternalURL }}) +{{ end -}} +{{- end }} diff --git a/src/cluster_mgr/config/grafana_config_v13.ini b/src/cluster_mgr/config/grafana_config_v13.ini new file mode 100644 index 00000000..cbf6a67b --- /dev/null +++ b/src/cluster_mgr/config/grafana_config_v13.ini @@ -0,0 +1,2 @@ +[server] +http_port = 3000 diff --git a/src/cluster_mgr/config/mono_alertmanager.yaml b/src/cluster_mgr/config/mono_alertmanager.yaml new file mode 100644 index 00000000..0ffecab9 --- /dev/null +++ b/src/cluster_mgr/config/mono_alertmanager.yaml @@ -0,0 +1,12 @@ +global: + resolve_timeout: 5m + +route: + group_by: ["alertname", "instance"] + group_wait: 10s + group_interval: 30s + repeat_interval: 10m + receiver: "null" + +receivers: + - name: "null" diff --git a/src/cluster_mgr/config/runtime_deps_ubuntu b/src/cluster_mgr/config/runtime_deps_ubuntu index eb4ee834..37b72c8a 100644 --- a/src/cluster_mgr/config/runtime_deps_ubuntu +++ b/src/cluster_mgr/config/runtime_deps_ubuntu @@ -5,3 +5,4 @@ curl unzip rsync rclone +musl diff --git a/src/cluster_mgr/src/cli/cmd_base.rs b/src/cluster_mgr/src/cli/cmd_base.rs index b39b6908..4663c151 100644 --- a/src/cluster_mgr/src/cli/cmd_base.rs +++ b/src/cluster_mgr/src/cli/cmd_base.rs @@ -3,24 +3,40 @@ use crate::cli::reconcile::{ }; use crate::cli::task::backup_utils::split_manifests; use crate::cli::task::download_task::DownloadTask; +use crate::cli::task::exec_custom_cmd::ExecCustomCommand; use crate::cli::task::group::Config; +use crate::cli::task::local_extract_task::LocalExtractTask; use crate::cli::task::monitor_ctl_task::MonitorCtlTask; use crate::cli::task::task_base::{ - set_verbose_task_output, TaskExecutionContext, TaskMgr, TaskResultPair, + set_verbose_task_output, TaskExecutionContext, TaskId, TaskMgr, TaskResultPair, }; +use crate::cli::task::task_controller::task_action_summary; use crate::cli::task::unpack_file_task::UnpackFileTask; -use crate::cli::task::upload::upload_task_builder::{upload_tasks, UploadTaskBuilderType}; +use crate::cli::task::upload::upload_task_builder::{ + build_task_instance, get_source_host, upload_tasks, UploadTaskBuilderType, +}; use crate::cli::util::{cpu_arch, file_pg_bar, os_id, os_major_version}; -use crate::cli::{upload_dir, SubCommand, UpdateMonitorComponent, HOME_DIR}; +use crate::cli::{ + download_dir, upload_dir, MonitorCommand, SubCommand, UpdateMonitorComponent, HOME_DIR, +}; use crate::cli::{BackupCommand, ProxyCommand}; -use crate::config::config_base::{DeployConfig, VersionRow}; -use crate::config::config_base::{GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, PROMETHEUS_FILE_KEY}; +use crate::config::config_base::{DeployConfig, UploadFile, VersionRow}; +use crate::config::config_base::{ + ALERTMANAGER_FILE_KEY, GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, PROMETHEUSALERT_FILE_KEY, + PROMETHEUS_FILE_KEY, +}; use crate::config::deployment::{version_digits, Deployment, Product}; +use crate::config::monitor::{ + Alertmanager, Exporter, Grafana, Monitor, Prometheus, ALERTMANAGER_WEBHOOK_ADAPTER_DEFAULT_URL, + ALERTMANAGER_WEBHOOK_ADAPTER_PORT, +}; use crate::config::proxy_config_base::ProxyConfig; use crate::config::storage_service_config::{ DataStoreServiceBackend, DataStoreServiceMode, RocksDB, RocksLocal, StorageService, }; -use crate::config::{DeploymentPackage, StorageProvider, CDN, CONFIG_PATH_DIR, UPLOAD_PATH_DIR}; +use crate::config::{ + DeploymentPackage, DownloadUrl, StorageProvider, CDN, CONFIG_PATH_DIR, UPLOAD_PATH_DIR, +}; use crate::github_release::{ fetch_eloqkv_releases, find_eloqkv_asset, find_product_asset, list_versions_from_releases, }; @@ -55,6 +71,10 @@ pub static NOT_PRINT_TASK_RESULT: &str = "NOT_PRINT_TASK_RESULT"; pub static HTTP_CLIENT: LazyLock = LazyLock::new(|| { reqwest::Client::builder() .connect_timeout(Duration::from_secs(30)) + .http1_only() + .pool_max_idle_per_host(0) + .pool_idle_timeout(Duration::from_secs(5)) + .tcp_keepalive(Duration::from_secs(30)) .build() .expect("can't init http client") }); @@ -67,6 +87,16 @@ pub static HTTP_INTERNAL: LazyLock = LazyLock::new(|| { .expect("can't init http client for internal use") }); +const DEFAULT_PROMETHEUS_URL: &str = + "https://github.com/prometheus/prometheus/releases/download/v2.42.0/prometheus-2.42.0.linux-amd64.tar.gz"; +const DEFAULT_ALERTMANAGER_URL: &str = + "https://github.com/prometheus/alertmanager/releases/download/v0.32.1/alertmanager-0.32.1.linux-amd64.tar.gz"; +const DEFAULT_GRAFANA_URL: &str = + "https://dl.grafana.com/oss/release/grafana-9.3.6.linux-amd64.tar.gz"; +const DEFAULT_PROMETHEUSALERT_URL: &str = ALERTMANAGER_WEBHOOK_ADAPTER_DEFAULT_URL; +const DEFAULT_NODE_EXPORTER_URL: &str = + "https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-amd64.tar.gz"; + pub struct CmdExecutor { task_mgr: Arc, state_mgr: Arc, @@ -165,6 +195,18 @@ impl Drop for ClusterMutationLock { } impl CmdExecutor { + fn release_asset_url( + product: &str, + version: &str, + store: &str, + os: &str, + arch: &str, + ) -> String { + format!( + "https://github.com/eloqdata/eloqkv/releases/download/{version}/{product}-{version}-{store}-{os}-{arch}.tar.gz" + ) + } + pub fn new(home: PathBuf) -> Self { Self { task_mgr: Arc::new(TaskMgr::new()), @@ -230,6 +272,24 @@ impl CmdExecutor { .collect_vec() } + fn summarize_monitor_status_rows(task_results: &[TaskResultPair]) -> Vec { + ObservedCluster::from_task_results("monitor".to_string(), task_results) + .services + .iter() + .filter(|service| { + matches!( + service.service.as_str(), + "prometheus" + | "alertmanager" + | "grafana" + | "alertmanager_webhook_adapter" + | "node_exporter" + ) + }) + .map(Self::status_row_from_observed) + .collect_vec() + } + fn status_row_from_observed(service: &ObservedServiceStatus) -> StatusSummaryRow { StatusSummaryRow { host: service.host.clone(), @@ -253,6 +313,26 @@ impl CmdExecutor { println!("{}", tabled::Table::new(rows)); } + fn print_monitor_status_summary(task_results: &[TaskResultPair]) { + let rows = Self::summarize_monitor_status_rows(task_results); + if rows.is_empty() { + println!("No monitor status rows found."); + return; + } + println!("{}", tabled::Table::new(rows)); + } + + fn print_monitor_log_hints() { + println!("Control-node execution log:"); + println!("\t~/.eloqctl/logs/last-monitor.log"); + println!("Remote component runtime logs:"); + println!("\tprometheus -> /tmp/eloq_prometheus.log"); + println!("\talertmanager -> /tmp/eloq_alertmanager.log"); + println!("\tgrafana -> /tmp/eloq_grafana_server.log"); + println!("\talertmanager-webhook-adapter -> /tmp/eloq_alertmanager_webhook_adapter.log"); + println!("\tnode_exporter -> /tmp/eloq_node_exporter.log"); + } + fn shell_single_quote(value: &str) -> String { format!("'{}'", value.replace('\'', "'\"'\"'")) } @@ -299,6 +379,16 @@ impl CmdExecutor { } else { Self::print_status_connect_hint(deploy_config, password.as_deref()); } + } else if let SubCommand::Monitor { + command: MonitorCommand::Status { .. }, + .. + } = cmd + { + Self::print_monitor_status_summary(task_results); + Self::print_monitor_log_hints(); + if !verbose { + println!("Tip: use `--verbose` to show per-operation execution details."); + } } } @@ -351,8 +441,14 @@ impl CmdExecutor { for node in &redis_nodes { let (host, port_str) = node.split_once(':').unwrap_or((node, "6379")); let port: u16 = port_str.parse().unwrap_or(6379); + let tls_enabled = config.deployment.tls_enabled(); let (endpoint_host, endpoint_port) = config.service_endpoint(host, port); - let probe = RedisProbe::with_password(endpoint_host, endpoint_port, password.clone()); + let probe = RedisProbe::with_password_and_tls( + endpoint_host, + endpoint_port, + password.clone(), + tls_enabled, + ); // Wait up to 120s for each node to respond to PING. probe.probe(120).await.map_err(|e| { anyhow!("cannot {operation}: node {node} is not serving Redis after waiting: {e}") @@ -460,7 +556,6 @@ impl CmdExecutor { | SubCommand::Restart { cluster } | SubCommand::UpdateConf { cluster, .. } | SubCommand::Remove { cluster, .. } - | SubCommand::Monitor { cluster, .. } | SubCommand::LogService { cluster, .. } | SubCommand::Update { cluster: Some(cluster), @@ -470,6 +565,9 @@ impl CmdExecutor { | SubCommand::ScaleLog { cluster, .. } | SubCommand::Backup { cluster, .. } | SubCommand::Failover { cluster, .. } => Some(cluster.clone()), + SubCommand::Monitor { cluster, command } => { + Self::monitor_cluster(cluster, command).ok() + } _ => None, } } @@ -490,56 +588,245 @@ impl CmdExecutor { self.home.join("download") } + fn monitor_cluster( + parent_cluster: &Option, + command: &MonitorCommand, + ) -> anyhow::Result { + let child_cluster = match command { + MonitorCommand::Start { cluster, .. } + | MonitorCommand::Stop { cluster, .. } + | MonitorCommand::Restart { cluster, .. } + | MonitorCommand::Status { cluster, .. } + | MonitorCommand::Update { cluster, .. } => cluster.clone(), + }; + + match (parent_cluster.clone(), child_cluster) { + (Some(parent), Some(child)) if parent != child => Err(anyhow!( + "monitor cluster is ambiguous: parent --cluster is '{}' but subcommand cluster is '{}'", + parent, child + )), + (Some(parent), _) => Ok(parent), + (_, Some(child)) => Ok(child), + (None, None) => Err(anyhow!( + "monitor cluster is required; use `eloqctl monitor --cluster ...` or `eloqctl monitor ... `" + )), + } + } + fn update_monitor_component_name(component: &UpdateMonitorComponent) -> &'static str { match component { UpdateMonitorComponent::Grafana => "grafana", UpdateMonitorComponent::Prometheus => "prometheus", + UpdateMonitorComponent::Alertmanager => "alertmanager", + UpdateMonitorComponent::Prometheusalert => "alertmanager-webhook-adapter", UpdateMonitorComponent::NodeExporter => "node_exporter", } } + fn monitor_update_component_key(component: &UpdateMonitorComponent) -> &'static str { + match component { + UpdateMonitorComponent::Grafana => GRAFANA_FILE_KEY, + UpdateMonitorComponent::Prometheus => PROMETHEUS_FILE_KEY, + UpdateMonitorComponent::Alertmanager => ALERTMANAGER_FILE_KEY, + UpdateMonitorComponent::Prometheusalert => PROMETHEUSALERT_FILE_KEY, + UpdateMonitorComponent::NodeExporter => NODE_EXPORTER_FILE_KEY, + } + } + + fn monitor_update_component_home(component: &UpdateMonitorComponent) -> &'static str { + match component { + UpdateMonitorComponent::Grafana => GRAFANA_FILE_KEY, + UpdateMonitorComponent::Prometheus => PROMETHEUS_FILE_KEY, + UpdateMonitorComponent::Alertmanager => ALERTMANAGER_FILE_KEY, + UpdateMonitorComponent::Prometheusalert => PROMETHEUSALERT_FILE_KEY, + UpdateMonitorComponent::NodeExporter => NODE_EXPORTER_FILE_KEY, + } + } + + fn monitor_update_uses_staged_dir(component: &UpdateMonitorComponent) -> bool { + matches!(component, UpdateMonitorComponent::Prometheusalert) + } + + fn monitor_update_components( + component: &UpdateMonitorComponent, + ) -> Vec { + match component { + UpdateMonitorComponent::Alertmanager => vec![ + UpdateMonitorComponent::Alertmanager, + UpdateMonitorComponent::Prometheusalert, + ], + other => vec![other.clone()], + } + } + + fn monitor_update_component_hosts( + &self, + config: &DeployConfig, + component: &UpdateMonitorComponent, + ) -> Vec { + let monitor_hosts = config.get_host_as_map(); + match component { + UpdateMonitorComponent::Grafana => monitor_hosts + .get(&DeploymentPackage::Grafana) + .cloned() + .unwrap_or_default(), + UpdateMonitorComponent::Prometheus => monitor_hosts + .get(&DeploymentPackage::Prometheus) + .cloned() + .unwrap_or_default(), + UpdateMonitorComponent::Alertmanager => monitor_hosts + .get(&DeploymentPackage::Alertmanager) + .cloned() + .unwrap_or_default(), + UpdateMonitorComponent::Prometheusalert => monitor_hosts + .get(&DeploymentPackage::PrometheusAlert) + .cloned() + .unwrap_or_default(), + UpdateMonitorComponent::NodeExporter => config.get_unique_host_list(), + } + } + + fn monitor_update_stop_tasks( + &self, + cmd: SubCommand, + config: &DeployConfig, + component: &UpdateMonitorComponent, + ) -> indexmap::IndexMap { + match component { + UpdateMonitorComponent::Grafana => MonitorCtlTask::grafana_ctl_task(cmd, config), + UpdateMonitorComponent::Prometheus => MonitorCtlTask::prometheus_ctl_task(cmd, config), + UpdateMonitorComponent::Alertmanager => { + MonitorCtlTask::alertmanager_ctl_task(cmd, config) + } + UpdateMonitorComponent::Prometheusalert => { + MonitorCtlTask::prometheusalert_ctl_task(cmd, config) + } + UpdateMonitorComponent::NodeExporter => MonitorCtlTask::exporter_ctl_task(cmd, config), + } + } + + fn monitor_default_host(deployment: &Deployment) -> String { + deployment + .monitor + .as_ref() + .and_then(|monitor| { + monitor + .prometheus + .as_ref() + .map(|component| component.host.clone()) + .or_else(|| { + monitor + .grafana + .as_ref() + .map(|component| component.host.clone()) + }) + .or_else(|| { + monitor + .alertmanager + .as_ref() + .map(|component| component.host.clone()) + }) + }) + .or_else(|| { + deployment + .tx_service + .tx_host_ports + .first() + .and_then(|host_port| host_port.split(':').next().map(ToOwned::to_owned)) + }) + .unwrap_or_else(|| "127.0.0.1".to_string()) + } + fn configure_monitor_update( config: &mut DeployConfig, component: &UpdateMonitorComponent, monitor_url: Option, + feishu_robot_urls: Vec, ) -> Result<()> { - let monitor = config.deployment.monitor.as_mut().ok_or_else(|| { - anyhow!( - "cluster '{}' has no monitor configuration", - config.deployment.cluster_name - ) - })?; + let default_host = Self::monitor_default_host(&config.deployment); + let prometheus_preferred_host = config + .deployment + .monitor + .as_ref() + .and_then(|monitor| { + monitor + .prometheus + .as_ref() + .map(|component| component.host.clone()) + }) + .unwrap_or_else(|| default_host.clone()); + let monitor = config.deployment.monitor.get_or_insert(Monitor { + data_dir: None, + prometheus: None, + alertmanager: None, + grafana: None, + node_exporter: None, + eloq_metrics: None, + }); match component { UpdateMonitorComponent::Grafana => { - let grafana = monitor.grafana.as_mut().ok_or_else(|| { - anyhow!( - "cluster '{}' has no grafana configuration", - config.deployment.cluster_name - ) - })?; + let grafana = monitor.grafana.get_or_insert_with(|| Grafana { + download_url: DEFAULT_GRAFANA_URL.to_string(), + port: 3301, + host: default_host.clone(), + }); if let Some(url) = monitor_url { grafana.download_url = url; } } UpdateMonitorComponent::Prometheus => { - let prometheus = monitor.prometheus.as_mut().ok_or_else(|| { - anyhow!( - "cluster '{}' has no prometheus configuration", - config.deployment.cluster_name - ) - })?; + let prometheus = monitor.prometheus.get_or_insert_with(|| Prometheus { + download_url: DEFAULT_PROMETHEUS_URL.to_string(), + port: 9500, + host: default_host.clone(), + retention_time: None, + retention_size: None, + remote_write_urls: None, + alertmanager_targets: None, + alert_thresholds: None, + }); if let Some(url) = monitor_url { prometheus.download_url = url; } } + UpdateMonitorComponent::Alertmanager => { + let alertmanager = monitor.alertmanager.get_or_insert_with(|| Alertmanager { + download_url: DEFAULT_ALERTMANAGER_URL.to_string(), + port: 9093, + host: prometheus_preferred_host.clone(), + feishu_robot_urls: None, + webhook_adapter_download_url: DEFAULT_PROMETHEUSALERT_URL.to_string(), + webhook_adapter_port: ALERTMANAGER_WEBHOOK_ADAPTER_PORT, + }); + if let Some(url) = monitor_url { + alertmanager.download_url = url; + } + if !feishu_robot_urls.is_empty() { + alertmanager.feishu_robot_urls = Some(feishu_robot_urls); + } + } + UpdateMonitorComponent::Prometheusalert => { + let alertmanager = monitor.alertmanager.get_or_insert_with(|| Alertmanager { + download_url: DEFAULT_ALERTMANAGER_URL.to_string(), + port: 9093, + host: prometheus_preferred_host.clone(), + feishu_robot_urls: None, + webhook_adapter_download_url: DEFAULT_PROMETHEUSALERT_URL.to_string(), + webhook_adapter_port: ALERTMANAGER_WEBHOOK_ADAPTER_PORT, + }); + if let Some(url) = monitor_url { + alertmanager.webhook_adapter_download_url = url; + } + if !feishu_robot_urls.is_empty() { + alertmanager.feishu_robot_urls = Some(feishu_robot_urls); + } + } UpdateMonitorComponent::NodeExporter => { - let exporter = monitor.node_exporter.as_mut().ok_or_else(|| { - anyhow!( - "cluster '{}' has no node_exporter configuration", - config.deployment.cluster_name - ) - })?; + let exporter = monitor.node_exporter.get_or_insert_with(|| Exporter { + url: DEFAULT_NODE_EXPORTER_URL.to_string(), + port: 9200, + }); if let Some(url) = monitor_url { exporter.url = url; } @@ -556,154 +843,155 @@ impl CmdExecutor { ) -> TaskExecutionContext { let mut executable = indexmap::IndexMap::new(); let mut barrier = vec![]; - - let download = DownloadTask::from_config(config).unwrap_or_default(); + let components = Self::monitor_update_components(component); + + let download = DownloadTask::instances(DownloadTask::from_urls( + components + .iter() + .map(|component| self.monitor_update_component_url(config, component)) + .collect(), + )); if !download.is_empty() { barrier.push(download.len()); executable.extend(download); } - let monitor_conf_upload = upload_tasks( - UploadTaskBuilderType::MonitorConf, - &Config::Cluster(config.clone()), - ); - if !monitor_conf_upload.is_empty() { - barrier.push(monitor_conf_upload.len()); - executable.extend(monitor_conf_upload); + let staged_entries = components + .iter() + .filter(|component| Self::monitor_update_uses_staged_dir(component)) + .map(|component| { + ( + Self::monitor_update_component_key(component).to_string(), + DownloadUrl::from_url_str( + &self.monitor_update_component_url(config, component), + ) + .unwrap(), + Self::monitor_update_component_home(component).to_string(), + ) + }) + .collect_vec(); + if !staged_entries.is_empty() { + let extract = LocalExtractTask::from_urls(staged_entries); + if !extract.is_empty() { + barrier.push(extract.len()); + executable.extend(extract); + } } - let monitor_hosts = config.get_host_as_map(); - let unpack_tasks = match component { - UpdateMonitorComponent::Grafana => monitor_hosts - .get(&DeploymentPackage::Grafana) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|host| { - UnpackFileTask::make_task_pair( - config, - &host, - config - .deployment - .monitor - .as_ref() - .unwrap() - .grafana - .as_ref() - .unwrap() - .download_url - .split('/') - .next_back() - .unwrap(), - GRAFANA_FILE_KEY, - vec![], - ) - }) - .collect::>(), - UpdateMonitorComponent::Prometheus => monitor_hosts - .get(&DeploymentPackage::Prometheus) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|host| { - UnpackFileTask::make_task_pair( - config, - &host, - config - .deployment - .monitor - .as_ref() - .unwrap() - .prometheus - .as_ref() - .unwrap() - .download_url - .split('/') - .next_back() - .unwrap(), - PROMETHEUS_FILE_KEY, - vec![], - ) - }) - .collect::>(), - UpdateMonitorComponent::NodeExporter => config - .get_unique_host_list() - .into_iter() - .map(|host| { - UnpackFileTask::make_task_pair( - config, - &host, - config - .deployment - .monitor - .as_ref() - .unwrap() - .node_exporter - .as_ref() - .unwrap() - .url - .split('/') - .next_back() - .unwrap(), - NODE_EXPORTER_FILE_KEY, - vec![], - ) - }) - .collect::>(), - }; - if !unpack_tasks.is_empty() { - barrier.push(unpack_tasks.len()); - executable.extend(unpack_tasks); + let mut upload_before_clean = indexmap::IndexMap::new(); + for component in &components { + if !Self::monitor_update_uses_staged_dir(component) { + upload_before_clean.extend(self.monitor_update_package_uploads(config, component)); + } + } + if !upload_before_clean.is_empty() { + barrier.push(upload_before_clean.len()); + executable.extend(upload_before_clean); } let stop_cmd = SubCommand::Monitor { - cluster: match cmd { + cluster: Some(match cmd { SubCommand::Update { cluster: Some(cluster), .. } => cluster.clone(), + SubCommand::Monitor { cluster, .. } => { + cluster.clone().expect("monitor cluster is resolved") + } _ => unreachable!(), + }), + command: MonitorCommand::Stop { + cluster: None, + components: vec![], }, - command: "stop".to_string(), }; let start_cmd = SubCommand::Monitor { - cluster: match cmd { + cluster: Some(match cmd { SubCommand::Update { cluster: Some(cluster), .. } => cluster.clone(), + SubCommand::Monitor { cluster, .. } => { + cluster.clone().expect("monitor cluster is resolved") + } _ => unreachable!(), + }), + command: MonitorCommand::Start { + cluster: None, + components: vec![], }, - command: "start".to_string(), }; - let stop_tasks = match component { - UpdateMonitorComponent::Grafana => { - MonitorCtlTask::grafana_ctl_task(stop_cmd.clone(), config) - } - UpdateMonitorComponent::Prometheus => { - MonitorCtlTask::prometheus_ctl_task(stop_cmd.clone(), config) - } - UpdateMonitorComponent::NodeExporter => { - MonitorCtlTask::exporter_ctl_task(stop_cmd.clone(), config) - } - }; + let mut stop_tasks = indexmap::IndexMap::new(); + for component in &components { + stop_tasks.extend(self.monitor_update_stop_tasks(stop_cmd.clone(), config, component)); + } if !stop_tasks.is_empty() { barrier.push(stop_tasks.len()); executable.extend(stop_tasks); } - let start_tasks = match component { - UpdateMonitorComponent::Grafana => { - MonitorCtlTask::grafana_ctl_task(start_cmd.clone(), config) - } - UpdateMonitorComponent::Prometheus => { - MonitorCtlTask::prometheus_ctl_task(start_cmd.clone(), config) + let mut clean_tasks = indexmap::IndexMap::new(); + for component in &components { + clean_tasks.extend(self.monitor_update_clean_tasks(config, component)); + } + if !clean_tasks.is_empty() { + barrier.push(clean_tasks.len()); + executable.extend(clean_tasks); + } + + let mut upload_after_clean = indexmap::IndexMap::new(); + for component in &components { + if Self::monitor_update_uses_staged_dir(component) { + upload_after_clean.extend(self.monitor_update_package_uploads(config, component)); } - UpdateMonitorComponent::NodeExporter => { - MonitorCtlTask::exporter_ctl_task(start_cmd.clone(), config) + } + if !upload_after_clean.is_empty() { + barrier.push(upload_after_clean.len()); + executable.extend(upload_after_clean); + } + + let mut unpack_tasks = indexmap::IndexMap::new(); + for component in &components { + if !Self::monitor_update_uses_staged_dir(component) { + unpack_tasks.extend( + self.monitor_update_component_hosts(config, component) + .into_iter() + .map(|host| { + UnpackFileTask::make_task_pair( + config, + &host, + &self.monitor_update_component_file_name(config, component), + Self::monitor_update_component_home(component), + vec![], + ) + }) + .collect::>(), + ); } - }; + } + if !unpack_tasks.is_empty() { + barrier.push(unpack_tasks.len()); + executable.extend(unpack_tasks); + } + + let monitor_conf_upload = upload_tasks( + UploadTaskBuilderType::MonitorConf, + &Config::Cluster(config.clone()), + ); + if !monitor_conf_upload.is_empty() { + barrier.push(monitor_conf_upload.len()); + executable.extend(monitor_conf_upload); + } + + let mut start_tasks = indexmap::IndexMap::new(); + for component in &components { + start_tasks.extend(self.monitor_update_stop_tasks( + start_cmd.clone(), + config, + component, + )); + } if !start_tasks.is_empty() { barrier.push(start_tasks.len()); executable.extend(start_tasks); @@ -719,6 +1007,155 @@ impl CmdExecutor { } } + fn monitor_update_package_uploads( + &self, + config: &DeployConfig, + component: &UpdateMonitorComponent, + ) -> indexmap::IndexMap< + crate::cli::task::task_base::TaskId, + crate::cli::task::task_base::TaskInstance, + > { + let file_name = self.monitor_update_component_file_name(config, component); + let file_key = Self::monitor_update_component_key(component); + let hosts = self.monitor_update_component_hosts(config, component); + let source = if Self::monitor_update_uses_staged_dir(component) { + LocalExtractTask::staged_dir_for( + &DownloadUrl::from_url_str(&self.monitor_update_component_url(config, component)) + .unwrap(), + Self::monitor_update_component_home(component), + ) + .to_string_lossy() + .to_string() + } else { + download_dir() + .join(&file_name) + .to_string_lossy() + .to_string() + }; + let source_host = get_source_host(None); + let config_ref = Config::Cluster(config.clone()); + + hosts + .into_iter() + .map(|host| { + let upload = UploadFile { + source: source.clone(), + dest: if Self::monitor_update_uses_staged_dir(component) { + format!( + "{}/{}", + config.install_dir(), + Self::monitor_update_component_home(component) + ) + } else { + format!("{}/{}", config.install_dir(), file_name) + }, + extension: if Self::monitor_update_uses_staged_dir(component) { + "dir".to_string() + } else { + file_name + .split('.') + .next_back() + .unwrap_or("pkg") + .to_string() + }, + host: host.clone(), + copy_dir: Self::monitor_update_uses_staged_dir(component), + delete_remote: Self::monitor_update_uses_staged_dir(component), + }; + build_task_instance( + source_host.clone(), + upload, + &config_ref, + "deploy", + &format!("upload_monitor_pkg_{}_{}", file_key, host), + ) + }) + .collect() + } + + fn monitor_update_clean_tasks( + &self, + config: &DeployConfig, + component: &UpdateMonitorComponent, + ) -> indexmap::IndexMap< + crate::cli::task::task_base::TaskId, + crate::cli::task::task_base::TaskInstance, + > { + let home = Self::monitor_update_component_home(component).to_string(); + let hosts = self.monitor_update_component_hosts(config, component); + let task_name = format!( + "clean_monitor_{}", + Self::update_monitor_component_name(component) + ); + + let clean_cmd = format!( + "rm -rf {install_dir}/{home} && mkdir -p {install_dir}/{home}", + install_dir = config.install_dir(), + ); + ExecCustomCommand::build_task_by_host( + clean_cmd, + &Config::Cluster(config.clone()), + hosts, + Some(task_name), + ) + } + + fn monitor_update_component_url( + &self, + config: &DeployConfig, + component: &UpdateMonitorComponent, + ) -> String { + match component { + UpdateMonitorComponent::Grafana => config + .deployment + .monitor + .as_ref() + .and_then(|m| m.grafana.as_ref()) + .map(|g| g.download_url.clone()) + .unwrap(), + UpdateMonitorComponent::Prometheus => config + .deployment + .monitor + .as_ref() + .and_then(|m| m.prometheus.as_ref()) + .map(|p| p.download_url.clone()) + .unwrap(), + UpdateMonitorComponent::Alertmanager => config + .deployment + .monitor + .as_ref() + .and_then(|m| m.alertmanager.as_ref()) + .map(|a| a.download_url.clone()) + .unwrap(), + UpdateMonitorComponent::Prometheusalert => config + .deployment + .monitor + .as_ref() + .and_then(|m| m.alertmanager.as_ref()) + .map(|a| a.webhook_adapter_download_url.clone()) + .unwrap(), + UpdateMonitorComponent::NodeExporter => config + .deployment + .monitor + .as_ref() + .and_then(|m| m.node_exporter.as_ref()) + .map(|n| n.url.clone()) + .unwrap(), + } + } + + fn monitor_update_component_file_name( + &self, + config: &DeployConfig, + component: &UpdateMonitorComponent, + ) -> String { + self.monitor_update_component_url(config, component) + .split('/') + .next_back() + .unwrap() + .to_string() + } + async fn run_update_command( &'static self, cmd: SubCommand, @@ -730,12 +1167,12 @@ impl CmdExecutor { unreachable!(); }; - let (download_only, monitor_component) = match &cmd { - SubCommand::Update { - download_only, - monitor, - .. - } => (*download_only, monitor.clone()), + let download_only = match &cmd { + SubCommand::Update { download_only, .. } => *download_only, + SubCommand::Monitor { + cluster: _, + command: MonitorCommand::Update { .. }, + } => false, _ => unreachable!(), }; @@ -768,8 +1205,12 @@ impl CmdExecutor { return Ok(()); } - if let Some(component) = monitor_component { - let context = self.monitor_update_context(&cmd, &cfg, &component); + if let SubCommand::Monitor { + cluster: _, + command: MonitorCommand::Update { component, .. }, + } = &cmd + { + let context = self.monitor_update_context(&cmd, &cfg, component); let outfile = if quiet { let f = fs::OpenOptions::new() .create(true) @@ -792,7 +1233,7 @@ impl CmdExecutor { println!( "cluster {} monitor component {} is updated!", cfg.deployment.cluster_name, - Self::update_monitor_component_name(&component) + Self::update_monitor_component_name(component) ); return Ok(()); } @@ -878,7 +1319,7 @@ impl CmdExecutor { let f = fs::OpenOptions::new() .create(true) .truncate(true) - .open(self.home.join("task-result"))?; + .open(self.home.join("operation-result"))?; Some(f) } else { None @@ -888,7 +1329,7 @@ impl CmdExecutor { task_mgr .write_task_result(outfile, verbose) .await - .expect("write task result failed"); + .expect("write operation result failed"); }); let rs = self @@ -942,7 +1383,7 @@ impl CmdExecutor { let f = fs::OpenOptions::new() .create(true) .truncate(true) - .open(self.home.join("task-result"))?; + .open(self.home.join("operation-result"))?; Some(f) } else { None @@ -952,7 +1393,7 @@ impl CmdExecutor { task_mgr .write_task_result(outfile, verbose) .await - .expect("write task result failed"); + .expect("write operation result failed"); }); let rs = self @@ -1381,8 +1822,11 @@ impl CmdExecutor { if plan.actions.contains(&ReconcileAction::RestartMonitor) { Box::pin(self.run_impl( SubCommand::Monitor { - command: "stop".to_string(), - cluster: cluster.clone(), + cluster: Some(cluster.clone()), + command: MonitorCommand::Stop { + cluster: None, + components: vec![], + }, }, None, quiet, @@ -1392,8 +1836,11 @@ impl CmdExecutor { .await?; Box::pin(self.run_impl( SubCommand::Monitor { - command: "start".to_string(), - cluster: cluster.clone(), + cluster: Some(cluster.clone()), + command: MonitorCommand::Start { + cluster: None, + components: vec![], + }, }, None, quiet, @@ -1483,10 +1930,6 @@ impl CmdExecutor { wait: _, detail: _, } - | SubCommand::Monitor { - command: _, - cluster, - } | SubCommand::Export { cluster, .. } | SubCommand::Remove { cluster, force: _ } | SubCommand::Connect { cluster } @@ -1518,13 +1961,8 @@ impl CmdExecutor { cluster: Some(cluster), version, download_only, - monitor, - monitor_url, .. } => { - if download_only && monitor.is_some() { - bail!("--download-only cannot be combined with --monitor"); - } if download_only && version.is_none() { bail!( "`update --download-only` requires a target version. Use `eloqctl update {cluster} --download-only`" @@ -1554,9 +1992,43 @@ impl CmdExecutor { } self.resolve_version(&mut config.deployment).await?; } - if let Some(component) = monitor.as_ref() { - Self::configure_monitor_update(&mut config, component, monitor_url.clone())?; - } + Self::validate_metrics_config(&config)?; + + Ok(Config::Cluster(config)) + } + SubCommand::Monitor { + cluster, + command: + ref command @ MonitorCommand::Update { + ref component, + ref url, + ref feishu_robot_url, + .. + }, + } => { + let cluster = Self::monitor_cluster(&cluster, command)?; + let mut config = self + .state_mgr + .load_deployment_from_state(&cluster) + .await? + .ok_or(anyhow!("cluster {} not found", cluster))?; + Self::configure_monitor_update( + &mut config, + component, + url.clone(), + feishu_robot_url.clone(), + )?; + Self::validate_metrics_config(&config)?; + + Ok(Config::Cluster(config)) + } + SubCommand::Monitor { cluster, command } => { + let cluster = Self::monitor_cluster(&cluster, &command)?; + let config = self + .state_mgr + .load_deployment_from_state(&cluster) + .await? + .ok_or(anyhow!("cluster {} not found", cluster))?; Self::validate_metrics_config(&config)?; Ok(Config::Cluster(config)) @@ -1692,6 +2164,9 @@ impl CmdExecutor { SubCommand::Update { cluster: Some(_), .. + } | SubCommand::Monitor { + cluster: _, + command: MonitorCommand::Update { .. }, } ) { let config = match option_config { @@ -1914,18 +2389,22 @@ impl CmdExecutor { ) .await?; if !failed_tasks.is_empty() { - let failed_hosts = failed_tasks - .iter() - .map(|t| t.task_host.clone()) - .unique() - .collect_vec(); eprintln!( - "Remove completed with {} failed task(s) on host(s): {}", - failed_tasks.len(), - failed_hosts.join(", ") + "Remove finished with {} cleanup task(s) still incomplete:", + failed_tasks.len() + ); + for task in &failed_tasks { + let action = serde_json::from_str::(&task.task) + .ok() + .map(|id| task_action_summary(&id.cmd, &id.task)) + .unwrap_or_else(|| "finish cleanup".to_string()); + eprintln!(" - {}: {}", task.task_host, action); + } + eprintln!( + "If those hosts are reachable, retry `eloqctl remove {cluster}`. Otherwise clean them manually." ); eprintln!( - "Please clean unreachable hosts manually, e.g.: rm -rf {}", + "Manual cleanup command: rm -rf {}", cfg.deployment.install_dir() ); } @@ -2143,32 +2622,47 @@ impl CmdExecutor { .storage_service .as_ref() .map_or("rocksdb".to_string(), |s| s.pretty_name()); - let releases = fetch_eloqkv_releases(&HTTP_CLIENT).await?; - if cnf.version.is_some() && cnf.version_str().eq_ignore_ascii_case("latest") { - let latest = - list_versions_from_releases(&releases, Some(cnf.product()), Some(store.as_str())) - .into_iter() - .map(|row: VersionRow| row.version) - .max_by(|a, b| version_digits(a).ok().cmp(&version_digits(b).ok())) - .ok_or_else(|| { - anyhow!("no available GitHub release found for store={store}") - })?; + let needs_latest_resolution = + cnf.version.is_some() && cnf.version_str().eq_ignore_ascii_case("latest"); + let releases = if needs_latest_resolution { + Some(fetch_eloqkv_releases(&HTTP_CLIENT).await?) + } else { + None + }; + + if needs_latest_resolution { + let latest = list_versions_from_releases( + releases.as_ref().unwrap(), + Some(cnf.product()), + Some(store.as_str()), + ) + .into_iter() + .map(|row: VersionRow| row.version) + .max_by(|a, b| version_digits(a).ok().cmp(&version_digits(b).ok())) + .ok_or_else(|| anyhow!("no available GitHub release found for store={store}"))?; info!("latest release version = {latest}"); cnf.version = Some(latest); } if cnf.tx_service.image.is_none() { let vers = cnf.version.as_deref().expect("version is missing"); - let img = find_eloqkv_asset(&releases, cnf.product(), vers, &store, &os, &arch)?.url; + let img = if let Some(releases) = releases.as_ref() { + find_eloqkv_asset(releases, cnf.product(), vers, &store, &os, &arch)?.url + } else { + Self::release_asset_url(cnf.product().name(), vers, &store, &os, &arch) + }; info!("tx service image is set: {img}"); cnf.tx_service.image = Some(img); } if let Some(logsrv) = &mut cnf.log_service { if logsrv.image.is_none() { let vers = cnf.version.as_deref().expect("version is missing"); - let img = - find_product_asset(&releases, "log-service", vers, &store, &os, &arch)?.url; + let img = if let Some(releases) = releases.as_ref() { + find_product_asset(releases, "log-service", vers, &store, &os, &arch)?.url + } else { + Self::release_asset_url("log-service", vers, &store, &os, &arch) + }; info!("log service image is set: {img}"); logsrv.image = Some(img); } diff --git a/src/cluster_mgr/src/cli/mod.rs b/src/cluster_mgr/src/cli/mod.rs index 75406c5a..67bd7cae 100644 --- a/src/cluster_mgr/src/cli/mod.rs +++ b/src/cluster_mgr/src/cli/mod.rs @@ -40,9 +40,101 @@ pub struct Command { pub enum UpdateMonitorComponent { Grafana, Prometheus, + Alertmanager, + #[value(hide = true)] + Prometheusalert, NodeExporter, } +#[derive(Clone, Debug, Hash, PartialEq, Eq, clap::ValueEnum)] +pub enum MonitorComponent { + Grafana, + Prometheus, + Alertmanager, + AlertmanagerWebhookAdapter, + NodeExporter, +} + +#[derive(Subcommand, Clone, Debug, Hash, PartialEq, Eq)] +pub enum MonitorCommand { + #[command( + long_about = "Start monitor components.\n\nExamples:\n eloqctl monitor start --cluster test-e2e\n eloqctl monitor start --cluster test-e2e --component grafana" + )] + Start { + #[arg(long = "cluster", help = "Existing cluster name")] + cluster: Option, + #[arg( + long = "component", + value_enum, + value_delimiter = ',', + help = "Monitor component(s) to operate on; omit to target all monitor components" + )] + components: Vec, + }, + #[command( + long_about = "Stop monitor components.\n\nExamples:\n eloqctl monitor stop --cluster test-e2e\n eloqctl monitor stop --cluster test-e2e --component alertmanager" + )] + Stop { + #[arg(long = "cluster", help = "Existing cluster name")] + cluster: Option, + #[arg( + long = "component", + value_enum, + value_delimiter = ',', + help = "Monitor component(s) to operate on; omit to target all monitor components" + )] + components: Vec, + }, + #[command( + long_about = "Restart monitor components.\n\nExamples:\n eloqctl monitor restart --cluster test-e2e\n eloqctl monitor restart --cluster test-e2e --component alertmanager-webhook-adapter" + )] + Restart { + #[arg(long = "cluster", help = "Existing cluster name")] + cluster: Option, + #[arg( + long = "component", + value_enum, + value_delimiter = ',', + help = "Monitor component(s) to operate on; omit to target all monitor components" + )] + components: Vec, + }, + #[command( + long_about = "Check monitor component status.\n\nExamples:\n eloqctl monitor status --cluster test-e2e\n eloqctl monitor status --cluster test-e2e --component grafana" + )] + Status { + #[arg(long = "cluster", help = "Existing cluster name")] + cluster: Option, + #[arg( + long = "component", + value_enum, + value_delimiter = ',', + help = "Monitor component(s) to inspect; omit to inspect all monitor components" + )] + components: Vec, + }, + #[command( + long_about = "Update one monitor component without touching EloqKV.\n\nExamples:\n eloqctl monitor update --cluster test-e2e --component grafana --url \n eloqctl monitor update --cluster test-e2e --component alertmanager --feishu-robot-url " + )] + Update { + #[arg(long = "cluster", help = "Existing cluster name")] + cluster: Option, + #[arg(long, value_enum, help = "Monitor component to update")] + component: UpdateMonitorComponent, + #[arg( + long, + help = "Override the tarball URL for the selected monitor component" + )] + url: Option, + #[arg( + long, + value_delimiter = ',', + help = "Configure one or more Feishu robot webhook URLs for Alertmanager notifications" + )] + feishu_robot_url: Vec, + }, +} + #[derive(Subcommand, Clone, Debug, Hash, PartialEq, Eq, AsRefStr)] #[command(next_line_help = true)] pub enum SubCommand { @@ -84,7 +176,12 @@ pub enum SubCommand { #[command(long_about = "Start cluster(TxService/LogService/Storage)")] Start { cluster: String, - #[arg(long, value_delimiter = ',', value_name = "nodes")] + #[arg( + long, + value_delimiter = ',', + value_name = "nodes", + value_parser = parse_host_port + )] nodes: Vec, }, #[command(long_about = "Stop cluster components")] @@ -105,7 +202,12 @@ pub enum SubCommand { all: bool, #[arg(long, value_name = "cluster password")] password: Option, - #[arg(long, value_delimiter = ',', value_name = "nodes")] + #[arg( + long, + value_delimiter = ',', + value_name = "nodes", + value_parser = parse_host_port + )] nodes: Vec, }, @@ -130,17 +232,12 @@ pub enum SubCommand { #[command(long_about = "Update an existing cluster to a target version.\n\ By default this performs a rolling upgrade that stops, replaces, and restarts services.\n\ Use `--download-only` to resolve the requested version and download the required EloqKV tarballs\n\ - into the local cache without changing remote hosts or cluster state.\n\ - Use `--monitor` to update only one monitor component without touching the EloqKV cluster.")] + into the local cache without changing remote hosts or cluster state.")] #[strum(serialize = "update")] Update { - #[arg( - help = "Existing cluster name; required for rolling updates, download-only, and monitor-only updates" - )] + #[arg(help = "Existing cluster name; required for rolling updates and download-only")] cluster: Option, - #[arg( - help = "Target EloqKV version or `latest`; required unless using --monitor without a version change" - )] + #[arg(help = "Target EloqKV version or `latest`")] version: Option, #[arg( long, @@ -148,17 +245,6 @@ pub enum SubCommand { help = "Download the target release tarballs into the local cache only; do not update remote hosts. Use as: `eloqctl update --download-only`" )] download_only: bool, - #[arg( - long, - value_enum, - help = "Update only the selected monitor component instead of the EloqKV cluster" - )] - monitor: Option, - #[arg( - long, - help = "Override the tarball URL for the selected monitor component; valid only with --monitor" - )] - monitor_url: Option, #[arg( long, value_name = "password for graceful shutdown and redis operations" @@ -247,9 +333,14 @@ pub enum SubCommand { #[command(long_about = "Run the SQLite schema script to create any missing tables")] Upgrade, - #[command(long_about = "monitor control component")] + #[command(long_about = "Manage monitor components without touching EloqKV")] #[strum(serialize = "monitor")] - Monitor { command: String, cluster: String }, + Monitor { + #[arg(long, global = true, help = "Existing cluster name")] + cluster: Option, + #[command(subcommand)] + command: MonitorCommand, + }, #[command(long_about = "LogService control component")] #[strum(serialize = "log-srv")] @@ -374,6 +465,25 @@ fn parse_duration(s: &str) -> Result { .map_err(|e| e.to_string()) } +fn parse_host_port(s: &str) -> Result { + let Some((host, port)) = s.split_once(':') else { + return Err(format!("invalid node '{s}': expected 'host:port'")); + }; + + if host.is_empty() { + return Err(format!("invalid node '{s}': host must not be empty")); + } + + if port.is_empty() { + return Err(format!("invalid node '{s}': port must not be empty")); + } + + port.parse::() + .map_err(|_| format!("invalid node '{s}': port must be a valid number"))?; + + Ok(s.to_string()) +} + fn parse_datetime(s: &str) -> Result, String> { // Try parsing with timezone information (RFC 3339) if let Ok(dt) = DateTime::parse_from_rfc3339(s) { diff --git a/src/cluster_mgr/src/cli/reconcile.rs b/src/cluster_mgr/src/cli/reconcile.rs index 7ff08306..a95904ab 100644 --- a/src/cluster_mgr/src/cli/reconcile.rs +++ b/src/cluster_mgr/src/cli/reconcile.rs @@ -118,7 +118,12 @@ fn parse_task_id_parts(task_id: &str) -> (String, String, String) { fn parse_monitor_status_task(task: &str) -> Option<(String, String)> { [ ("prometheus-status-", "prometheus"), + ("alertmanager-status-", "alertmanager"), ("grafana-status-", "grafana"), + ( + "alertmanager-webhook-adapter-status-", + "alertmanager_webhook_adapter", + ), ("node_exporter-status-", "node_exporter"), ] .into_iter() diff --git a/src/cluster_mgr/src/cli/task/check_task.rs b/src/cluster_mgr/src/cli/task/check_task.rs index e701ee54..90d56876 100644 --- a/src/cluster_mgr/src/cli/task/check_task.rs +++ b/src/cluster_mgr/src/cli/task/check_task.rs @@ -122,6 +122,51 @@ impl CheckTask { Ok(None) } + async fn check_alertmanager(&self, host: TaskHost) -> Result> { + let ssh_k = self.config.connection.ssh_auth_key().unwrap(); + let sess = ssh::SSHSession::from_task_host(host, ssh_k).await?; + let port = self + .config + .deployment + .monitor + .as_ref() + .unwrap() + .alertmanager + .as_ref() + .unwrap() + .port; + for p in sess.used_tcp_ports().await? { + if port == p { + bail!("alertmanager socket {}:{p} is already used", self.host); + } + } + Ok(None) + } + + async fn check_prometheusalert(&self, host: TaskHost) -> Result> { + let ssh_k = self.config.connection.ssh_auth_key().unwrap(); + let sess = ssh::SSHSession::from_task_host(host, ssh_k).await?; + let port = self + .config + .deployment + .monitor + .as_ref() + .unwrap() + .alertmanager + .as_ref() + .unwrap() + .webhook_adapter_port; + for p in sess.used_tcp_ports().await? { + if port == p { + bail!( + "alertmanager-webhook-adapter socket {}:{p} is already used", + self.host + ); + } + } + Ok(None) + } + async fn check_proxy(&self, _: TaskHost) -> Result> { Ok(None) } @@ -151,7 +196,9 @@ impl TaskExecutor for CheckTask { Ok(None) } DeploymentPackage::Prometheus => self.check_prometheus(host).await, + DeploymentPackage::Alertmanager => self.check_alertmanager(host).await, DeploymentPackage::Grafana => self.check_grafana(host).await, + DeploymentPackage::PrometheusAlert => self.check_prometheusalert(host).await, DeploymentPackage::EloqLog => self.check_log_sv(host).await, DeploymentPackage::Proxy => self.check_proxy(host).await, } diff --git a/src/cluster_mgr/src/cli/task/download_task.rs b/src/cluster_mgr/src/cli/task/download_task.rs index d07ddf3d..a53b4070 100644 --- a/src/cluster_mgr/src/cli/task/download_task.rs +++ b/src/cluster_mgr/src/cli/task/download_task.rs @@ -9,22 +9,28 @@ use crate::cli::{download_dir, CMD, CMD_OUTPUT, CMD_STATUS}; use crate::config::config_base::DeployConfig; use crate::config::proxy_config_base::ProxyConfig; use crate::config::DownloadUrl; -use anyhow::{anyhow, Ok, Result}; +use anyhow::{anyhow, Result}; use futures::stream::StreamExt; use indexmap::IndexMap; use indicatif::{MultiProgress, ProgressBar}; use itertools::Itertools; -use reqwest::header::{ACCEPT, USER_AGENT}; +use reqwest::header::{ACCEPT, CONNECTION, RANGE, USER_AGENT}; +use reqwest::StatusCode; use serde::Deserialize; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::ffi::{OsStr, OsString}; use std::fs; +use std::fs::OpenOptions; use std::io::Read; use std::io::Write; use std::path::PathBuf; +use std::time::Duration; use tracing::{error, info}; +const MAX_DOWNLOAD_ATTEMPTS: usize = 4; +const DOWNLOAD_RETRY_DELAY: Duration = Duration::from_secs(2); + #[derive(Debug, Clone)] pub struct DownloadTask { task_id: TaskId, @@ -240,6 +246,7 @@ impl TaskExecutor for DownloadTask { // Try HEAD first to check remote content-length for cache validation let remote_len = HTTP_CLIENT .head(&self.url) + .header(CONNECTION, "close") .send() .await .ok() @@ -275,62 +282,168 @@ impl TaskExecutor for DownloadTask { ); return Ok(None); } - } - let response = HTTP_CLIENT - .get(&self.url) - .send() - .await - .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; - let status = response.status(); - if !status.is_success() { - error!("Download falied http status_code = {:?}", status.as_str()); - return Err(anyhow!(DownloadErr(self.url.clone(), status.to_string()))); - } - let file_len = response.content_length().unwrap(); - - // Double-check cache after successful GET (in case of concurrent download) - if save_path.exists() { - let digest_matches = match &expected_digest { - Some(expected) => Self::sha256_file(&save_path) - .map(|actual| actual == *expected) - .unwrap_or(false), - None => false, - }; - if digest_matches - || (expected_digest.is_none() - && fs::metadata(&save_path) - .map(|m| m.len() == file_len) - .unwrap_or(false)) + if expected_digest.is_none() + && remote_len.is_none() + && fs::metadata(&save_path) + .map(|m| m.len() > 0) + .unwrap_or(false) { - info!("local file cache {:?} found.", save_path); + info!( + "local file cache {:?} found (remote HEAD unavailable), reusing existing file.", + save_path + ); return Ok(None); } } - // TODO(zhanghao): Use HTTP range header to resume download + let part_path = append_ext(save_path.clone(), "partial"); - let mut part_file = std::fs::File::create(part_path.as_path()) - .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + let mut final_file_len = remote_len.unwrap_or(0); + for attempt in 1..=MAX_DOWNLOAD_ATTEMPTS { + let mut resume_from = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0); + + if let Some(expected_len) = remote_len { + if resume_from > expected_len { + fs::remove_file(&part_path).ok(); + resume_from = 0; + } else if resume_from == expected_len && expected_len > 0 { + if let Some(expected) = &expected_digest { + let actual = Self::sha256_file(&part_path) + .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + if &actual == expected { + fs::rename(&part_path, save_path.as_path()) + .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + info!("partial download {:?} already complete.", save_path); + return Ok(None); + } + fs::remove_file(&part_path).ok(); + resume_from = 0; + } else { + fs::rename(&part_path, save_path.as_path()) + .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + info!("partial download {:?} already complete.", save_path); + return Ok(None); + } + } + } - // start downloading - self.pg_bar.set_length(file_len); - self.pg_bar - .set_message(format!("{} Downloading...", self.name)); - let mut stream_reader = response.bytes_stream(); - while let Some(stream_chunk) = stream_reader.next().await { - if let Err(err) = stream_chunk { - error!("DownloadRemote task error file={},msg={}", url, err); - return Err(anyhow!(DownloadErr(url.clone(), err.to_string()))); + let mut request = HTTP_CLIENT.get(&self.url).header(CONNECTION, "close"); + if resume_from > 0 { + request = request.header(RANGE, format!("bytes={resume_from}-")); } - let chunk = stream_chunk.unwrap(); - if let Err(write_err) = part_file.write_all(&chunk) { - error!("DownloadTask {} write local file error {} ", url, write_err); - return Err(anyhow!(DownloadErr(url.clone(), write_err.to_string()))); + let response = match request.send().await { + Ok(response) => response, + Err(err) => { + let err_msg = err.to_string(); + if attempt < MAX_DOWNLOAD_ATTEMPTS { + info!( + "download attempt {attempt}/{MAX_DOWNLOAD_ATTEMPTS} failed to start for {}: {}", + self.name, err_msg + ); + tokio::time::sleep(DOWNLOAD_RETRY_DELAY).await; + continue; + } + return Err(anyhow!(DownloadErr(url.clone(), err_msg))); + } + }; + let status = response.status(); + if !(status.is_success() || status == StatusCode::PARTIAL_CONTENT) { + error!("Download falied http status_code = {:?}", status.as_str()); + return Err(anyhow!(DownloadErr(self.url.clone(), status.to_string()))); + } + + let (mut part_file, file_len, resumed) = + if resume_from > 0 && status == StatusCode::PARTIAL_CONTENT { + let total_len = remote_len + .unwrap_or_else(|| resume_from + response.content_length().unwrap_or(0)); + let file = OpenOptions::new() + .create(true) + .append(true) + .open(part_path.as_path()) + .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + (file, total_len, true) + } else { + if resume_from > 0 { + fs::remove_file(&part_path).ok(); + } + let file = std::fs::File::create(part_path.as_path()) + .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + ( + file, + response.content_length().unwrap_or(remote_len.unwrap_or(0)), + false, + ) + }; + final_file_len = file_len; + + // Double-check cache after successful GET (in case of concurrent download) + if save_path.exists() && file_len > 0 { + let digest_matches = match &expected_digest { + Some(expected) => Self::sha256_file(&save_path) + .map(|actual| actual == *expected) + .unwrap_or(false), + None => false, + }; + if digest_matches + || (expected_digest.is_none() + && fs::metadata(&save_path) + .map(|m| m.len() == file_len) + .unwrap_or(false)) + { + info!("local file cache {:?} found.", save_path); + return Ok(None); + } + } + + self.pg_bar.set_length(file_len); + if resumed { + self.pg_bar.set_position(resume_from); + } else { + self.pg_bar.set_position(0); + } + self.pg_bar + .set_message(format!("{} Downloading...", self.name)); + + let mut stream_reader = response.bytes_stream(); + let mut stream_failed = None; + while let Some(stream_chunk) = stream_reader.next().await { + if let Err(err) = stream_chunk { + error!("DownloadRemote task error file={},msg={}", url, err); + stream_failed = Some(err.to_string()); + break; + } + let chunk = stream_chunk.unwrap(); + if let Err(write_err) = part_file.write_all(&chunk) { + error!("DownloadTask {} write local file error {} ", url, write_err); + return Err(anyhow!(DownloadErr(url.clone(), write_err.to_string()))); + } + self.pg_bar.inc(chunk.len() as u64); + } + drop(part_file); + + let current_len = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0); + let err_msg = if let Some(err) = stream_failed { + err + } else if file_len > 0 && current_len < file_len { + format!( + "incomplete download after attempt {attempt}: expected {file_len} bytes, got {current_len}" + ) + } else { + self.pg_bar + .finish_with_message(format!("{} Downloaded!", self.name)); + break; + }; + + if attempt < MAX_DOWNLOAD_ATTEMPTS { + info!( + "download attempt {attempt}/{MAX_DOWNLOAD_ATTEMPTS} incomplete for {} (have {current_len}/{file_len} bytes), retrying", + self.name + ); + tokio::time::sleep(DOWNLOAD_RETRY_DELAY).await; + } else { + return Err(anyhow!(DownloadErr(url.clone(), err_msg))); } - self.pg_bar.inc(chunk.len() as u64); } - self.pg_bar - .finish_with_message(format!("{} Downloaded!", self.name)); if let Some(expected) = &expected_digest { let actual = Self::sha256_file(&part_path) @@ -344,6 +457,18 @@ impl TaskExecutor for DownloadTask { } } + if final_file_len > 0 { + let actual_len = fs::metadata(&part_path) + .map(|m| m.len()) + .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; + if actual_len != final_file_len { + return Err(anyhow!(DownloadErr( + self.url.clone(), + format!("download size mismatch: expected {final_file_len}, got {actual_len}") + ))); + } + } + fs::rename(part_path, save_path.as_path()) .map_err(|err| DownloadErr(url.clone(), err.to_string()))?; diff --git a/src/cluster_mgr/src/cli/task/eloq_tx_ctl_task.rs b/src/cluster_mgr/src/cli/task/eloq_tx_ctl_task.rs index d2229148..367f4804 100644 --- a/src/cluster_mgr/src/cli/task/eloq_tx_ctl_task.rs +++ b/src/cluster_mgr/src/cli/task/eloq_tx_ctl_task.rs @@ -129,6 +129,7 @@ pub struct RedisProbe { host: String, port: u16, password: Option, + tls_enabled: bool, } impl RedisProbe { @@ -137,6 +138,7 @@ impl RedisProbe { host, port, password: None, + tls_enabled: false, } } @@ -145,17 +147,38 @@ impl RedisProbe { host, port, password, + tls_enabled: false, + } + } + + pub fn with_password_and_tls( + host: String, + port: u16, + password: Option, + tls_enabled: bool, + ) -> Self { + Self { + host, + port, + password, + tls_enabled, } } pub async fn probe(&self, mut wait_secs: i32) -> anyhow::Result { info!("Probe whether Redis is ready to serve requests"); + let scheme = if self.tls_enabled { "rediss" } else { "redis" }; let url = if let Some(ref pass) = self.password { - format!("redis://:{pass}@{}:{}/", self.host, self.port) + format!("{scheme}://:{pass}@{}:{}/", self.host, self.port) + } else { + format!("{scheme}://{}:{}/", self.host, self.port) + }; + let client_url = if self.tls_enabled { + format!("{url}#insecure") } else { - format!("redis://{}:{}/", self.host, self.port) + url.clone() }; - let client = redis::Client::open(url.clone())?; + let client = redis::Client::open(client_url)?; loop { match client.get_connection() { Ok(mut con) => { @@ -512,6 +535,12 @@ impl EloqTxCtlTask { receiver, } } + + fn redis_probe_endpoint(&self, host: &str, port: u16) -> (String, u16, bool) { + let tls_enabled = self.config.deployment.tls_enabled(); + let (endpoint_host, endpoint_port) = self.config.service_endpoint(host, port); + (endpoint_host, endpoint_port, tls_enabled) + } } fn extract_server_type_and_port(input: &str) -> (&str, &str) { @@ -550,6 +579,7 @@ impl TaskExecutor for EloqTxCtlTask { if let Some(receiver) = &self.receiver { let timeout = Duration::from_secs(5); let start_time = tokio::time::Instant::now(); + let allow_missing_topology = matches!(self.ctl_cmd.as_ref(), "stop" | "force_stop"); loop { if receiver.has_changed().unwrap_or(false) { @@ -572,6 +602,13 @@ impl TaskExecutor for EloqTxCtlTask { } if start_time.elapsed() >= timeout { + if allow_missing_topology { + info!( + "Timeout waiting for cluster topology for {}. Falling back to direct process control.", + self.task_id.format_string() + ); + break; + } let redis_cmd_result = HashMap::from([ ( CMD.to_string(), @@ -619,6 +656,7 @@ impl TaskExecutor for EloqTxCtlTask { let (host_value, user) = ssh_session.ssh_conn_info(); let check_status_cmd = eloq_cmd_with_port!(TxCtlCmd::Status, tx_bin, user, port).cmd_value(); + let status_error_cmd = check_status_cmd.clone(); let check_status = eloq_cmd_with_port!(TxCtlCmd::Status, tx_bin, user.to_string(), port); let cmd_val = check_status.cmd_value(); @@ -629,29 +667,51 @@ impl TaskExecutor for EloqTxCtlTask { "status" => { let wait_secs = TaskArgValue::into_inner_value::(task_arg.get(WAIT_SECS).unwrap().clone()); - if wait_secs >= 0 && server_type == "txservice" { + if wait_secs >= 0 && matches!(server_type, "txservice" | "standby") { let rp = self.config.redis_password(None); - if self.config.deployment.cluster_mode.unwrap_or(false) { - let mut startup_nodes = - self.config.get_host_port_list(DeploymentPackage::EloqTx); - startup_nodes.extend( - self.config - .get_host_port_list(DeploymentPackage::EloqStandby), - ); - let startup_nodes = self.config.service_host_ports(startup_nodes); - RedisProbe::probe_cluster(startup_nodes, rp, wait_secs).await + let cs_port: u16 = port.parse().unwrap(); + let (endpoint_host, endpoint_port, tls_enabled) = + self.redis_probe_endpoint(&self.task_id.host, cs_port); + let probe_host = if endpoint_host == self.task_id.host { + host_value } else { - let cs_port: u16 = port.parse().unwrap(); - let (endpoint_host, endpoint_port) = - self.config.service_endpoint(&self.task_id.host, cs_port); - let probe_host = if endpoint_host == self.task_id.host { - host_value - } else { - endpoint_host - }; - RedisProbe::with_password(probe_host, endpoint_port, rp) - .probe(wait_secs) - .await + endpoint_host + }; + let probe_result = RedisProbe::with_password_and_tls( + probe_host, + endpoint_port, + rp, + tls_enabled, + ) + .probe(wait_secs) + .await?; + let status_code = probe_result + .get(CMD_STATUS) + .map(|v| TaskArgValue::into_inner_value::(v.clone())) + .unwrap_or(0); + if status_code == 0 { + Ok(probe_result) + } else { + let detail = probe_result + .get(CMD_OUTPUT) + .map(|v| TaskArgValue::into_inner_value::(v.clone())) + .unwrap_or_else(|| "probe failed".to_string()); + Ok(HashMap::from([ + ( + CMD.to_string(), + TaskArgValue::Str(format!( + "Dial Redis={}://{}:{}", + if tls_enabled { "rediss" } else { "redis" }, + self.task_id.host, + endpoint_port + )), + ), + (CMD_STATUS.to_string(), TaskArgValue::Number(0)), + ( + CMD_OUTPUT.to_string(), + TaskArgValue::Str(format!("eloqkv service is down: {detail}")), + ), + ])) } } else { check_process_status @@ -729,9 +789,21 @@ impl TaskExecutor for EloqTxCtlTask { let start_cmd = self.ctl_cmd.cmd_value(); info!("start_cmd: {}", start_cmd); info!("check_status_cmd: {}", check_status_cmd); - tx_ctl!(self, check_process_status, {==, PID_NOT_FOUND}, async || -> anyhow::Result { - wait_command_complete!(start_cmd, check_status_cmd, ssh_session.clone(), is_some) - }) + let start_cmd_for_wait = start_cmd.clone(); + let mut process_started = tx_ctl!(self, check_process_status, {==, PID_NOT_FOUND}, async || -> anyhow::Result { + wait_command_complete!(start_cmd_for_wait, check_status_cmd, ssh_session.clone(), is_some) + })?; + if let Some(output) = process_started.get(CMD_OUTPUT) { + let start_output = TaskArgValue::into_inner_value::(output.clone()); + process_started.insert(CMD.to_string(), TaskArgValue::Str(start_cmd)); + process_started.insert( + CMD_OUTPUT.to_string(), + TaskArgValue::Str(format!( + "start process check={start_output}, cluster readiness is verified by later status/topology steps" + )), + ); + } + Ok(process_started) } _ => { unreachable!("Unrecognized command: {}", ctl_cmd_ref); @@ -739,7 +811,21 @@ impl TaskExecutor for EloqTxCtlTask { }; ssh_session.close().await?; - let mut ctl_rtn_value = eloq_ctl_rs?; + let mut ctl_rtn_value = match eloq_ctl_rs { + Ok(value) => value, + Err(err) if ctl_cmd_ref == "status" => HashMap::from([ + (CMD.to_string(), TaskArgValue::Str(status_error_cmd)), + (CMD_STATUS.to_string(), TaskArgValue::Number(0)), + ( + CMD_OUTPUT.to_string(), + TaskArgValue::Str(format!("eloqkv service is down: {err}")), + ), + ]), + Err(err) => return Err(err), + }; + if ctl_cmd_ref == "status" { + ctl_rtn_value.insert(CMD_STATUS.to_string(), TaskArgValue::Number(0)); + } if ctl_cmd_ref == "status" && ctl_rtn_value.contains_key(PROCESS_PID) { let pid = TaskArgValue::into_inner_value::( ctl_rtn_value.get(PROCESS_PID).unwrap().clone(), diff --git a/src/cluster_mgr/src/cli/task/group/backup_group.rs b/src/cluster_mgr/src/cli/task/group/backup_group.rs index b249806e..24876d19 100644 --- a/src/cluster_mgr/src/cli/task/group/backup_group.rs +++ b/src/cluster_mgr/src/cli/task/group/backup_group.rs @@ -272,7 +272,7 @@ impl TaskGroup for BackupTaskGroup { }); } } else { - println!("No backups found to delete."); + println!("No backups found for this cluster; skipping backup cleanup."); return Ok(TaskExecutionContext { task_group: "backup".to_string(), barrier: Some(vec![]), diff --git a/src/cluster_mgr/src/cli/task/group/check_group.rs b/src/cluster_mgr/src/cli/task/group/check_group.rs index 66ca9638..a7b03668 100644 --- a/src/cluster_mgr/src/cli/task/group/check_group.rs +++ b/src/cluster_mgr/src/cli/task/group/check_group.rs @@ -65,12 +65,24 @@ impl TaskGroup for CheckTaskGroup { input, executable ); + make_check_tasks!( + DeploymentPackage::Alertmanager, + cluster_config, + input, + executable + ); make_check_tasks!( DeploymentPackage::Grafana, cluster_config, input, executable ); + make_check_tasks!( + DeploymentPackage::PrometheusAlert, + cluster_config, + input, + executable + ); } Ok(TaskExecutionContext { task_group: cmd_arg.as_ref().to_string(), diff --git a/src/cluster_mgr/src/cli/task/group/db_cluster_ctrl_group.rs b/src/cluster_mgr/src/cli/task/group/db_cluster_ctrl_group.rs index f2b7a63c..fba4ee61 100644 --- a/src/cluster_mgr/src/cli/task/group/db_cluster_ctrl_group.rs +++ b/src/cluster_mgr/src/cli/task/group/db_cluster_ctrl_group.rs @@ -7,10 +7,10 @@ use crate::cli::task::group::{Config, CtrlDBTaskGroup, MonitorCtlTaskGroup, Task use crate::cli::task::monitor_ctl_task::MonitorCtlTask; use crate::cli::task::redis_op_task::{ClusterNodes, RedisOpTask}; use crate::cli::task::task_base::{TaskExecutionContext, TaskHost, TaskId, TaskInstance}; -use crate::cli::task::task_utils::{stop_with_failover, stop_with_hot_standby}; +use crate::cli::task::task_utils::stop_with_hot_standby; use crate::cli::task::topology_display_task::TopologyDisplayTask; use crate::cli::task::topology_update_task::TopologyUpdateTask; -use crate::cli::SubCommand; +use crate::cli::{MonitorCommand, SubCommand}; use crate::config::config_base::DeployConfig; use crate::config::deployment::Product; use crate::config::DeploymentPackage; @@ -41,8 +41,11 @@ impl TaskGroup for CtrlDBTaskGroup { .await; if monitor && cluster_config.deployment.monitor.is_some() { let stop_moni = SubCommand::Monitor { - cluster: cluster.clone(), - command: "stop".to_string(), + cluster: Some(cluster.clone()), + command: MonitorCommand::Stop { + cluster: None, + components: vec![], + }, }; let TaskExecutionContext { task_group: _, @@ -156,8 +159,11 @@ impl TaskGroup for CtrlDBTaskGroup { .await; if monitor && cluster_config.deployment.monitor.is_some() { let stop_moni = SubCommand::Monitor { - cluster: cluster.clone(), - command: "stop".to_string(), + cluster: Some(cluster.clone()), + command: MonitorCommand::Stop { + cluster: None, + components: vec![], + }, }; let TaskExecutionContext { task_group: _, @@ -270,7 +276,15 @@ impl CtrlDBTaskGroup { }; if has_nodes { - stop_with_failover(cmd.clone(), config, &mut barrier, &mut executable).await; + let stop_nodes = EloqTxCtlTask::from_config_with_channel( + cmd.clone(), + config, + ServerType::Node, + None, + ) + .expect("stop nodes error"); + barrier.push(stop_nodes.len()); + executable.extend(stop_nodes); } else if is_from_remove || is_force_stop { // Enter this branch when: // - The user explicitly requests to remove or force-stop the cluster. @@ -546,6 +560,22 @@ impl CtrlDBTaskGroup { ) -> IndexMap { let deployment = &config.deployment; let mut executable = IndexMap::new(); + let tx_status_cmd = match &cmd { + SubCommand::Status { + cluster, + user, + password, + wait, + detail, + } => SubCommand::Status { + cluster: cluster.clone(), + user: user.clone(), + password: password.clone(), + wait: Some(wait.unwrap_or(0)), + detail: *detail, + }, + _ => cmd.clone(), + }; // DSS status (when rocksdb is EloqDssRocksdb or DataStoreService Remote mode, only if not external) if let Some(storage) = &deployment.storage_service { @@ -570,20 +600,23 @@ impl CtrlDBTaskGroup { } if config.deployment.tx_service.standby_host_ports.is_some() { let status_standby = - EloqTxCtlTask::from_config(cmd.clone(), config, ServerType::Standby); + EloqTxCtlTask::from_config(tx_status_cmd.clone(), config, ServerType::Standby); executable.extend(status_standby); } if config.deployment.tx_service.voter_host_ports.is_some() { let status_voter = EloqTxCtlTask::from_config(cmd.clone(), config, ServerType::Voter); executable.extend(status_voter); } - let status_tx = EloqTxCtlTask::from_config(cmd.clone(), config, ServerType::Tx); + let status_tx = EloqTxCtlTask::from_config(tx_status_cmd, config, ServerType::Tx); executable.extend(status_tx); if deployment.monitor.is_some() { let monitor_status_cmd = SubCommand::Monitor { - cluster: deployment.cluster_name.clone(), - command: "status".to_string(), + cluster: Some(deployment.cluster_name.clone()), + command: MonitorCommand::Status { + cluster: None, + components: vec![], + }, }; executable.extend(MonitorCtlTask::exporter_ctl_task( monitor_status_cmd.clone(), @@ -593,7 +626,18 @@ impl CtrlDBTaskGroup { monitor_status_cmd.clone(), config, )); - executable.extend(MonitorCtlTask::grafana_ctl_task(monitor_status_cmd, config)); + executable.extend(MonitorCtlTask::alertmanager_ctl_task( + monitor_status_cmd.clone(), + config, + )); + executable.extend(MonitorCtlTask::grafana_ctl_task( + monitor_status_cmd.clone(), + config, + )); + executable.extend(MonitorCtlTask::prometheusalert_ctl_task( + monitor_status_cmd, + config, + )); } executable diff --git a/src/cluster_mgr/src/cli/task/group/launch_group.rs b/src/cluster_mgr/src/cli/task/group/launch_group.rs index fe459dac..9b376ab2 100644 --- a/src/cluster_mgr/src/cli/task/group/launch_group.rs +++ b/src/cluster_mgr/src/cli/task/group/launch_group.rs @@ -13,7 +13,7 @@ use crate::cli::task::task_base::{ }; use crate::cli::task::topology_display_task::TopologyDisplayTask; use crate::cli::task::topology_update_task::TopologyUpdateTask; -use crate::cli::SubCommand; +use crate::cli::{MonitorCommand, SubCommand}; use crate::config::CONFIG_PATH_DIR; use std::collections::HashMap; use std::env; @@ -189,8 +189,11 @@ impl TaskGroup for LaunchTaskGroup { MonitorCtlTaskGroup .tasks( SubCommand::Monitor { - cluster: cluster_config.deployment.cluster_name.clone(), - command: "start".to_string(), + cluster: Some(cluster_config.deployment.cluster_name.clone()), + command: MonitorCommand::Start { + cluster: None, + components: vec![], + }, }, config, ) diff --git a/src/cluster_mgr/src/cli/task/group/monitor_ctl_group.rs b/src/cluster_mgr/src/cli/task/group/monitor_ctl_group.rs index 117f4b5c..e0f6692c 100644 --- a/src/cluster_mgr/src/cli/task/group/monitor_ctl_group.rs +++ b/src/cluster_mgr/src/cli/task/group/monitor_ctl_group.rs @@ -1,10 +1,24 @@ use crate::cli::task::group::{Config, MonitorCtlTaskGroup, TaskGroup}; use crate::cli::task::monitor_ctl_task::MonitorCtlTask; use crate::cli::task::task_base::TaskExecutionContext; -use crate::cli::task::upload::upload_task_builder::{upload_tasks, UploadTaskBuilderType}; -use crate::cli::SubCommand; +use crate::cli::task::upload::monitor_upload_builder::MonitorInfraConfUploadBuilder; +use crate::cli::{MonitorCommand, MonitorComponent, SubCommand}; use indexmap::IndexMap; +fn selected_components(command: &MonitorCommand) -> &[MonitorComponent] { + match command { + MonitorCommand::Start { components, .. } + | MonitorCommand::Stop { components, .. } + | MonitorCommand::Restart { components, .. } + | MonitorCommand::Status { components, .. } => components.as_slice(), + MonitorCommand::Update { .. } => &[], + } +} + +fn component_enabled(selected: &[MonitorComponent], component: MonitorComponent) -> bool { + selected.is_empty() || selected.contains(&component) +} + #[async_trait::async_trait] impl TaskGroup for MonitorCtlTaskGroup { async fn tasks( @@ -29,39 +43,132 @@ impl TaskGroup for MonitorCtlTaskGroup { }); } let monitor_ctl_cmd = match &cmd_arg { - SubCommand::Monitor { - cluster: _, - command, - } => command, + SubCommand::Monitor { command, .. } => command, _ => unreachable!(), }; let mut executable = IndexMap::new(); let mut barrier = vec![]; - let exporter_task_instance = - MonitorCtlTask::exporter_ctl_task(cmd_arg.clone(), cluster_config); - let prometheus_task_instance = - MonitorCtlTask::prometheus_ctl_task(cmd_arg.clone(), cluster_config); - let grafana_task_instance = - MonitorCtlTask::grafana_ctl_task(cmd_arg.clone(), cluster_config); + let selected = selected_components(monitor_ctl_cmd); - if monitor_ctl_cmd.to_lowercase().eq("start") { + let start_like = matches!( + monitor_ctl_cmd, + MonitorCommand::Start { .. } | MonitorCommand::Restart { .. } + ); + if start_like { // Re-upload the Prometheus config (and other monitor config files) before // starting the services so that any config changes (remote_write_urls, // retention_time, etc.) are applied to the running Prometheus instance. - let monitor_conf_upload = upload_tasks(UploadTaskBuilderType::MonitorConf, config); + let monitor_conf_upload = + MonitorInfraConfUploadBuilder::build_for_cmd(config, "monitor"); if !monitor_conf_upload.is_empty() { barrier.push(monitor_conf_upload.len()); executable.extend(monitor_conf_upload); } } - barrier.push(exporter_task_instance.len()); - barrier.push(prometheus_task_instance.len()); - barrier.push(grafana_task_instance.len()); + let cluster_name = match &cmd_arg { + SubCommand::Monitor { cluster, .. } => { + cluster.clone().expect("monitor cluster is resolved") + } + _ => unreachable!(), + }; + let stop_cmd = SubCommand::Monitor { + cluster: Some(cluster_name.clone()), + command: MonitorCommand::Stop { + cluster: None, + components: selected.to_vec(), + }, + }; + let start_cmd = SubCommand::Monitor { + cluster: Some(cluster_name), + command: MonitorCommand::Start { + cluster: None, + components: selected.to_vec(), + }, + }; + let component_cmd = if matches!(monitor_ctl_cmd, MonitorCommand::Restart { .. }) { + stop_cmd.clone() + } else { + cmd_arg.clone() + }; - executable.extend(exporter_task_instance); - executable.extend(prometheus_task_instance); - executable.extend(grafana_task_instance); + let exporter_task_instance = if component_enabled(selected, MonitorComponent::NodeExporter) + { + MonitorCtlTask::exporter_ctl_task(component_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }; + let prometheus_task_instance = if component_enabled(selected, MonitorComponent::Prometheus) + { + MonitorCtlTask::prometheus_ctl_task(component_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }; + let alertmanager_task_instance = + if component_enabled(selected, MonitorComponent::Alertmanager) { + MonitorCtlTask::alertmanager_ctl_task(component_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }; + let grafana_task_instance = if component_enabled(selected, MonitorComponent::Grafana) { + MonitorCtlTask::grafana_ctl_task(component_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }; + let adapter_task_instance = + if component_enabled(selected, MonitorComponent::AlertmanagerWebhookAdapter) { + MonitorCtlTask::prometheusalert_ctl_task(component_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }; + + let phase_sets = vec![ + exporter_task_instance, + prometheus_task_instance, + alertmanager_task_instance, + grafana_task_instance, + adapter_task_instance, + ]; + + for tasks in phase_sets { + barrier.push(tasks.len()); + executable.extend(tasks); + } + + if matches!(monitor_ctl_cmd, MonitorCommand::Restart { .. }) { + let restart_sets = vec![ + if component_enabled(selected, MonitorComponent::NodeExporter) { + MonitorCtlTask::exporter_ctl_task(start_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }, + if component_enabled(selected, MonitorComponent::Prometheus) { + MonitorCtlTask::prometheus_ctl_task(start_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }, + if component_enabled(selected, MonitorComponent::Alertmanager) { + MonitorCtlTask::alertmanager_ctl_task(start_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }, + if component_enabled(selected, MonitorComponent::Grafana) { + MonitorCtlTask::grafana_ctl_task(start_cmd.clone(), cluster_config) + } else { + IndexMap::new() + }, + if component_enabled(selected, MonitorComponent::AlertmanagerWebhookAdapter) { + MonitorCtlTask::prometheusalert_ctl_task(start_cmd, cluster_config) + } else { + IndexMap::new() + }, + ]; + + for tasks in restart_sets { + barrier.push(tasks.len()); + executable.extend(tasks); + } + } let cmd_ref = cmd_arg.as_ref(); Ok(TaskExecutionContext { diff --git a/src/cluster_mgr/src/cli/task/group/remove_group.rs b/src/cluster_mgr/src/cli/task/group/remove_group.rs index aa703ef3..c5ece93f 100644 --- a/src/cluster_mgr/src/cli/task/group/remove_group.rs +++ b/src/cluster_mgr/src/cli/task/group/remove_group.rs @@ -64,7 +64,7 @@ impl TaskGroup for RemoveTaskGroup { .map(|(host, dirs)| { let content = dirs .into_iter() - .map(|d| format!("rm -r {}", d)) + .map(|d| format!("rm -rf {}", d)) .join(" && "); let task_host = TaskHost::remote(&cluster_config.connection, &host); @@ -96,7 +96,7 @@ impl TaskGroup for RemoveTaskGroup { let clean_tasks = ExecCustomCommand::from_config( &cmd_arg, "clean", - format!("rm -r {}", cluster_config.install_dir()), + format!("rm -rf {}", cluster_config.install_dir()), config, ); barrier.push(clean_tasks.len()); diff --git a/src/cluster_mgr/src/cli/task/local_extract_task.rs b/src/cluster_mgr/src/cli/task/local_extract_task.rs index 6d15467c..b711e26e 100644 --- a/src/cluster_mgr/src/cli/task/local_extract_task.rs +++ b/src/cluster_mgr/src/cli/task/local_extract_task.rs @@ -1,23 +1,28 @@ use crate::cli::task::task_base::{ CmdErr, ExecutionValue, TaskArgValue, TaskExecutor, TaskHost, TaskId, TaskInstance, }; +use crate::cli::util::file_pg_bar; use crate::cli::{download_dir, CMD, CMD_OUTPUT, CMD_STATUS}; use crate::config::config_base::{ - DeployConfig, ELOQ_FILE_KEY, ELOQ_LOG_FILE_KEY, GRAFANA_FILE_KEY, LOG_SERVICE_HOME, - NODE_EXPORTER_FILE_KEY, PROMETHEUS_FILE_KEY, + DeployConfig, ALERTMANAGER_FILE_KEY, ELOQ_FILE_KEY, ELOQ_LOG_FILE_KEY, GRAFANA_FILE_KEY, + LOG_SERVICE_HOME, NODE_EXPORTER_FILE_KEY, PROMETHEUSALERT_FILE_KEY, PROMETHEUS_FILE_KEY, }; +use crate::config::monitor::ALERTMANAGER_WEBHOOK_ADAPTER_BINARY; use crate::config::DownloadUrl; use anyhow::{anyhow, Result}; use async_trait::async_trait; use flate2::read::GzDecoder; use indexmap::IndexMap; +use indicatif::{MultiProgress, ProgressBar}; use itertools::Itertools; use std::collections::HashMap; use std::fs; use std::fs::File; -use std::path::{Path, PathBuf}; +use std::io; +use std::path::{Component, Path, PathBuf}; use tar::Archive; use tracing::info; +use zip::ZipArchive; const ARCHIVE_PATH: &str = "_archive_path"; const EXTRACT_ROOT: &str = "_extract_root"; @@ -26,6 +31,7 @@ const UNPACK_DEST: &str = "_unpack_dest"; #[derive(Clone, Debug)] pub struct LocalExtractTask { task_id: TaskId, + pg_bar: ProgressBar, } impl LocalExtractTask { @@ -47,6 +53,7 @@ impl LocalExtractTask { } fn instances(entries: Vec<(String, DownloadUrl, String)>) -> IndexMap { + let mpg_bar = MultiProgress::new(); entries .into_iter() .unique_by(|(_key, url, dest)| (url.file_name(), dest.clone())) @@ -57,6 +64,7 @@ impl LocalExtractTask { task: format!("{}_extract", file_name), host: "_local".to_string(), }; + let pg_bar = mpg_bar.add(file_pg_bar()); let archive_path = Self::cached_archive_path(&url); let extract_root = Self::extract_root_for(&url); ( @@ -73,7 +81,7 @@ impl LocalExtractTask { ), (UNPACK_DEST.to_string(), TaskArgValue::Str(unpack_dest)), ]), - task: Box::new(Self::new(task_id)), + task: Box::new(Self::new(task_id, pg_bar)), task_host: TaskHost::Local, }, ) @@ -81,8 +89,8 @@ impl LocalExtractTask { .collect() } - pub fn new(task_id: TaskId) -> Self { - Self { task_id } + pub fn new(task_id: TaskId, pg_bar: ProgressBar) -> Self { + Self { task_id, pg_bar } } pub fn cached_archive_path(url: &DownloadUrl) -> PathBuf { @@ -98,7 +106,7 @@ impl LocalExtractTask { DownloadUrl::Remote(_) => PathBuf::from(url.cache_dir().unwrap()), }; path.push("_extracted"); - path.push(url.file_name().trim_end_matches(".tar.gz")); + path.push(Self::archive_stem(&url.file_name())); path } @@ -111,15 +119,64 @@ impl LocalExtractTask { ELOQ_FILE_KEY => Some(config.deployment.product().home().to_string()), ELOQ_LOG_FILE_KEY => Some(LOG_SERVICE_HOME.to_string()), PROMETHEUS_FILE_KEY => Some("prometheus".to_string()), + ALERTMANAGER_FILE_KEY => Some("alertmanager".to_string()), GRAFANA_FILE_KEY => Some("grafana".to_string()), + PROMETHEUSALERT_FILE_KEY => Some(PROMETHEUSALERT_FILE_KEY.to_string()), NODE_EXPORTER_FILE_KEY => Some("node_exporter".to_string()), _ => None, } } - fn unpack_archive(archive_path: &Path, destination: &Path) -> Result<()> { + fn unpack_archive(&self, archive_path: &Path, destination: &Path) -> Result<()> { + let file_name = archive_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + if file_name.ends_with(".zip") { + return self.unpack_zip_archive(archive_path, destination); + } + if file_name.ends_with(".tar.gz") { + return self.unpack_tar_gz_archive(archive_path, destination); + } + self.unpack_raw_binary(archive_path, destination) + } + + fn unpack_raw_binary(&self, archive_path: &Path, destination: &Path) -> Result<()> { let file = File::open(archive_path)?; - let decoder = GzDecoder::new(file); + let binary_len = file.metadata()?.len(); + self.pg_bar.set_length(binary_len); + self.pg_bar.set_position(0); + self.pg_bar.set_message(format!( + "{} Extracting...", + Self::archive_display_name(archive_path) + )); + let mut reader = self.pg_bar.wrap_read(file); + let output_path = destination.join(ALERTMANAGER_WEBHOOK_ADAPTER_BINARY); + let mut out_file = File::create(&output_path)?; + io::copy(&mut reader, &mut out_file)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&output_path, fs::Permissions::from_mode(0o755))?; + } + self.pg_bar.finish_with_message(format!( + "{} Extracted!", + Self::archive_display_name(archive_path) + )); + Ok(()) + } + + fn unpack_tar_gz_archive(&self, archive_path: &Path, destination: &Path) -> Result<()> { + let file = File::open(archive_path)?; + let compressed_len = file.metadata()?.len(); + self.pg_bar.set_length(compressed_len); + self.pg_bar.set_position(0); + self.pg_bar.set_message(format!( + "{} Extracting...", + Self::archive_display_name(archive_path) + )); + let reader = self.pg_bar.wrap_read(file); + let decoder = GzDecoder::new(reader); let mut archive = Archive::new(decoder); for entry in archive.entries()? { @@ -135,8 +192,117 @@ impl LocalExtractTask { } entry.unpack(&out_path)?; } + self.pg_bar.finish_with_message(format!( + "{} Extracted!", + Self::archive_display_name(archive_path) + )); + Ok(()) + } + + #[cfg(unix)] + fn apply_post_extract_permissions(destination: &Path, unpack_dest: &str) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if unpack_dest != PROMETHEUSALERT_FILE_KEY { + return Ok(()); + } + + for binary_name in [ + "PrometheusAlert", + "zabbixclient", + ALERTMANAGER_WEBHOOK_ADAPTER_BINARY, + ] { + let binary_path = destination.join(binary_name); + if !binary_path.exists() { + continue; + } + let metadata = fs::metadata(&binary_path)?; + let current_mode = metadata.permissions().mode() & 0o777; + if current_mode & 0o111 != 0 { + continue; + } + fs::set_permissions( + &binary_path, + fs::Permissions::from_mode(current_mode | 0o111), + )?; + } + Ok(()) + } + + #[cfg(not(unix))] + fn apply_post_extract_permissions(_destination: &Path, _unpack_dest: &str) -> Result<()> { + Ok(()) + } + + fn unpack_zip_archive(&self, archive_path: &Path, destination: &Path) -> Result<()> { + let file = File::open(archive_path)?; + let compressed_len = file.metadata()?.len(); + self.pg_bar.set_length(compressed_len); + self.pg_bar.set_position(0); + self.pg_bar.set_message(format!( + "{} Extracting...", + Self::archive_display_name(archive_path) + )); + let reader = self.pg_bar.wrap_read(file); + let mut archive = ZipArchive::new(reader)?; + + for idx in 0..archive.len() { + let mut entry = archive.by_index(idx)?; + let Some(safe_name) = entry.enclosed_name().map(|p| p.to_path_buf()) else { + continue; + }; + let stripped = Self::strip_first_component(&safe_name); + if stripped.as_os_str().is_empty() { + continue; + } + let out_path = destination.join(stripped); + if entry.is_dir() { + fs::create_dir_all(&out_path)?; + continue; + } + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent)?; + } + let mut out_file = File::create(&out_path)?; + io::copy(&mut entry, &mut out_file)?; + #[cfg(unix)] + if let Some(mode) = entry.unix_mode() { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&out_path, fs::Permissions::from_mode(mode))?; + } + } + + self.pg_bar.finish_with_message(format!( + "{} Extracted!", + Self::archive_display_name(archive_path) + )); Ok(()) } + + fn archive_display_name(archive_path: &Path) -> String { + archive_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("archive") + .to_string() + } + + fn archive_stem(file_name: &str) -> &str { + file_name + .strip_suffix(".tar.gz") + .or_else(|| file_name.strip_suffix(".zip")) + .unwrap_or(file_name) + } + + fn strip_first_component(path: &Path) -> PathBuf { + path.components() + .skip(1) + .filter_map(|component| match component { + Component::Normal(part) => Some(part), + _ => None, + }) + .collect() + } } #[async_trait] @@ -172,7 +338,8 @@ impl TaskExecutor for LocalExtractTask { fs::remove_dir_all(&extract_root)?; } fs::create_dir_all(&destination)?; - Self::unpack_archive(&archive_path, &destination)?; + self.unpack_archive(&archive_path, &destination)?; + Self::apply_post_extract_permissions(&destination, &unpack_dest)?; let mut result = HashMap::new(); result.insert( @@ -187,3 +354,101 @@ impl TaskExecutor for LocalExtractTask { Ok(Some(result)) } } + +#[cfg(test)] +mod tests { + use super::LocalExtractTask; + use crate::config::config_base::PROMETHEUSALERT_FILE_KEY; + use indicatif::ProgressBar; + use std::fs; + use std::fs::File; + use std::io::Write; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + use zip::write::SimpleFileOptions; + + fn temp_dir(name: &str) -> PathBuf { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("cluster-mgr-{name}-{suffix}")); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn unzip_preserves_unix_executable_mode() { + let root = temp_dir("zip-mode"); + let archive_path = root.join("linux.zip"); + let dest = root.join("dest"); + fs::create_dir_all(&dest).unwrap(); + + let file = File::create(&archive_path).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let opts = SimpleFileOptions::default().unix_permissions(0o755); + writer.add_directory("linux/", opts).unwrap(); + writer.start_file("linux/PrometheusAlert", opts).unwrap(); + writer.write_all(b"#!/bin/sh\nexit 0\n").unwrap(); + writer.finish().unwrap(); + + let task = LocalExtractTask::new( + crate::cli::task::task_base::TaskId { + cmd: "extract".to_string(), + task: "linux.zip_extract".to_string(), + host: "_local".to_string(), + }, + ProgressBar::hidden(), + ); + task.unpack_archive(&archive_path, &dest).unwrap(); + + let extracted = dest.join("PrometheusAlert"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(extracted).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o755); + } + } + + #[test] + fn unzip_webhook_adapter_without_unix_mode_still_becomes_executable() { + let root = temp_dir("zip-webhook-adapter-no-mode"); + let archive_path = root.join("linux.zip"); + let extract_root = root.join("extract-root"); + let destination = extract_root.join(PROMETHEUSALERT_FILE_KEY); + fs::create_dir_all(&destination).unwrap(); + + let file = File::create(&archive_path).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let opts = SimpleFileOptions::default(); + writer.add_directory("linux/", opts).unwrap(); + writer.start_file("linux/PrometheusAlert", opts).unwrap(); + writer.write_all(b"binary").unwrap(); + writer.start_file("linux/zabbixclient", opts).unwrap(); + writer.write_all(b"helper").unwrap(); + writer.finish().unwrap(); + + let task = LocalExtractTask::new( + crate::cli::task::task_base::TaskId { + cmd: "extract".to_string(), + task: "linux.zip_extract".to_string(), + host: "_local".to_string(), + }, + ProgressBar::hidden(), + ); + task.unpack_archive(&archive_path, &destination).unwrap(); + LocalExtractTask::apply_post_extract_permissions(&destination, PROMETHEUSALERT_FILE_KEY) + .unwrap(); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for binary_name in ["PrometheusAlert", "zabbixclient"] { + let extracted = destination.join(binary_name); + let mode = fs::metadata(extracted).unwrap().permissions().mode() & 0o777; + assert_ne!(mode & 0o111, 0, "{binary_name} should be executable"); + } + } + } +} diff --git a/src/cluster_mgr/src/cli/task/mod.rs b/src/cluster_mgr/src/cli/task/mod.rs index 66f3899a..def74ac7 100644 --- a/src/cluster_mgr/src/cli/task/mod.rs +++ b/src/cluster_mgr/src/cli/task/mod.rs @@ -44,3 +44,4 @@ pub mod topology_update_task; mod tx_conf_update_task; pub mod unpack_file_task; pub mod upload; +pub mod wait_replica_ready_task; diff --git a/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs b/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs index 5c6b9b24..ffedda2b 100644 --- a/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs +++ b/src/cluster_mgr/src/cli/task/monitor_ctl_task.rs @@ -4,30 +4,72 @@ use crate::cli::task::task_base::CmdErr; use crate::cli::task::task_base::{ ExecutionValue, TaskArgValue, TaskExecutor, TaskHost, TaskId, TaskInstance, }; -use crate::cli::task::task_utils::{ - check_pid, ctl_action_wait_complete, parse_process_pid, PID_NOT_FOUND, PROCESS_PID, -}; -use crate::cli::{SubCommand, CMD_OUTPUT}; +use crate::cli::task::task_utils::{check_pid, parse_process_pid, PID_NOT_FOUND, PROCESS_PID}; +use crate::cli::{MonitorCommand, SubCommand, CMD_OUTPUT}; use crate::config::config_base::{ - DeployConfig, GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, PROMETHEUS_FILE_KEY, + DeployConfig, ALERTMANAGER_FILE_KEY, GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, + PROMETHEUSALERT_FILE_KEY, PROMETHEUS_FILE_KEY, +}; +use crate::config::monitor::{Monitor, ALERTMANAGER_WEBHOOK_ADAPTER_BINARY}; +use crate::config::{ + DeploymentPackage, ALERTMANAGER_CONFIG_FILE, GRAFANA_CONFIG_FILE, PROMETHEUS_CONFIG_FILE, }; -use crate::config::monitor::Monitor; -use crate::config::DeploymentPackage; -use crate::config::PROMETHEUS_CONFIG_FILE; -use crate::{task_return_value, wait_command_complete}; +use crate::task_return_value; use indexmap::IndexMap; use itertools::Itertools; use std::collections::HashMap; +use std::time::Duration; use tracing::{debug, info}; #[derive(Clone, Debug)] pub enum MonitorComponentCommand { NodeExporter { home: String }, Prometheus { home: String }, + Alertmanager { home: String }, Grafana { home: String }, + PrometheusAlert { home: String }, } impl MonitorComponentCommand { + fn service_name(&self) -> &'static str { + match self { + MonitorComponentCommand::NodeExporter { .. } => "node_exporter", + MonitorComponentCommand::Prometheus { .. } => "prometheus", + MonitorComponentCommand::Alertmanager { .. } => "alertmanager", + MonitorComponentCommand::Grafana { .. } => "grafana", + MonitorComponentCommand::PrometheusAlert { .. } => "alertmanager_webhook_adapter", + } + } + + fn log_path(&self) -> &'static str { + match self { + MonitorComponentCommand::NodeExporter { .. } => "/tmp/eloq_node_exporter.log", + MonitorComponentCommand::Prometheus { .. } => "/tmp/eloq_prometheus.log", + MonitorComponentCommand::Alertmanager { .. } => "/tmp/eloq_alertmanager.log", + MonitorComponentCommand::Grafana { .. } => "/tmp/eloq_grafana_server.log", + MonitorComponentCommand::PrometheusAlert { .. } => { + "/tmp/eloq_alertmanager_webhook_adapter.log" + } + } + } + + fn executable_paths(&self) -> Vec { + match self { + MonitorComponentCommand::NodeExporter { home } => vec![format!("{home}/node_exporter")], + MonitorComponentCommand::Prometheus { home } => vec![format!("{home}/prometheus")], + MonitorComponentCommand::Alertmanager { home } => { + vec![format!("{home}/alertmanager")] + } + MonitorComponentCommand::Grafana { home } => vec![ + format!("{home}/bin/grafana-server"), + format!("{home}/bin/grafana"), + ], + MonitorComponentCommand::PrometheusAlert { home } => { + vec![format!("{home}/{ALERTMANAGER_WEBHOOK_ADAPTER_BINARY}")] + } + } + } + pub fn start(&self, monitor: &Monitor) -> Option { match self { MonitorComponentCommand::NodeExporter { home } => { @@ -40,12 +82,22 @@ impl MonitorComponentCommand { }) } MonitorComponentCommand::Grafana { home } => { - // Assuming Grafana is always configured when this command is used Some(format!( - r#"{home}/bin/grafana-server -homepath {home} -config {home}/conf/defaults.ini > /tmp/eloq_grafana_server.log 2>&1 &"#, - home = home + r#"if [ -x {home}/bin/grafana-server ]; then {home}/bin/grafana-server -homepath {home} -config {home}/conf/{config}; else {home}/bin/grafana server -homepath {home} -config {home}/conf/{config}; fi > /tmp/eloq_grafana_server.log 2>&1 &"#, + home = home, + config = GRAFANA_CONFIG_FILE )) } + MonitorComponentCommand::Alertmanager { home } => monitor.alertmanager.as_ref().map( + |alertmanager| { + format!( + r#"{home}/alertmanager --config.file={home}/{config} --storage.path={home}/data --web.listen-address=0.0.0.0:{port} > /tmp/eloq_alertmanager.log 2>&1 &"#, + home = home, + config = ALERTMANAGER_CONFIG_FILE, + port = alertmanager.port + ) + }, + ), MonitorComponentCommand::Prometheus { home } => { monitor.prometheus.as_ref().map(|prom| { let retention_flags = prom.retention_flags(); @@ -62,6 +114,17 @@ impl MonitorComponentCommand { ) }) } + MonitorComponentCommand::PrometheusAlert { home } => monitor + .alertmanager + .as_ref() + .map(|alertmanager| { + format!( + r#"sh -c 'cd {home} && exec {home}/{binary} --listen-address=0.0.0.0:{port} --tmpl-dir {home}/templates --tmpl-lang zh --signature EloqKV' > /tmp/eloq_alertmanager_webhook_adapter.log 2>&1 &"#, + home = home, + binary = ALERTMANAGER_WEBHOOK_ADAPTER_BINARY, + port = alertmanager.webhook_adapter_port + ) + }), } } @@ -69,6 +132,10 @@ impl MonitorComponentCommand { format!("kill {pid}") } + pub fn force_stop(&self, pid: String) -> String { + format!("kill -9 {pid}") + } + pub fn reload_url(&self, monitor: &Monitor) -> Option { match self { MonitorComponentCommand::Prometheus { .. } => monitor @@ -83,7 +150,9 @@ impl MonitorComponentCommand { let monitor_component_home = match &self { MonitorComponentCommand::NodeExporter { home } => home, MonitorComponentCommand::Prometheus { home } => home, + MonitorComponentCommand::Alertmanager { home } => home, MonitorComponentCommand::Grafana { home } => home, + MonitorComponentCommand::PrometheusAlert { home } => home, }; format!("ps -e -o pid,cmd --no-headers | grep \"{monitor_component_home}\" | grep -v grep") } @@ -108,6 +177,7 @@ mod tests { alertmanager_targets: None, alert_thresholds: None, }), + alertmanager: None, grafana: None, node_exporter: None, eloq_metrics: None, @@ -133,6 +203,108 @@ pub struct MonitorCtlTask { } impl MonitorCtlTask { + async fn start_failure_details(&self, ssh_session: SSHSession) -> String { + let mut sections = Vec::new(); + let log_cmd = format!( + "tail -n 40 {} 2>/dev/null || echo ''", + self.monitor_ctl.log_path() + ); + if let Ok(log_rs) = ssh_session.command(&log_cmd, CollectOutput).await { + let output = TaskArgValue::into_inner_value::( + log_rs + .get(CMD_OUTPUT) + .cloned() + .unwrap_or(TaskArgValue::Str(String::new())), + ) + .trim() + .to_string(); + if !output.is_empty() { + sections.push(format!("startup log:\n{output}")); + } + } + + let files = self.monitor_ctl.executable_paths(); + if !files.is_empty() { + let inspect_cmd = format!("ls -l {}", files.join(" ")); + if let Ok(file_rs) = ssh_session.command(&inspect_cmd, CollectOutput).await { + let output = TaskArgValue::into_inner_value::( + file_rs + .get(CMD_OUTPUT) + .cloned() + .unwrap_or(TaskArgValue::Str(String::new())), + ) + .trim() + .to_string(); + if !output.is_empty() { + sections.push(format!("file modes:\n{output}")); + } + } + } + + if sections.is_empty() { + format!( + "no additional diagnostics captured for {}", + self.monitor_ctl.service_name() + ) + } else { + sections.join("\n") + } + } + + async fn wait_started( + ssh_session: SSHSession, + process_info_cmd: String, + ) -> anyhow::Result { + for _ in 0..15 { + let process_rs = check_pid( + process_info_cmd.clone(), + ssh_session.clone(), + parse_process_pid, + ) + .await?; + let pid = TaskArgValue::into_inner_value::( + process_rs.get(PROCESS_PID).unwrap().clone(), + ); + if pid != PID_NOT_FOUND { + return Ok(process_rs); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + check_pid(process_info_cmd, ssh_session, parse_process_pid).await + } + + async fn wait_stopped( + ssh_session: SSHSession, + process_info_cmd: String, + ) -> anyhow::Result { + for _ in 0..15 { + let process_rs = check_pid( + process_info_cmd.clone(), + ssh_session.clone(), + parse_process_pid, + ) + .await?; + let pid = TaskArgValue::into_inner_value::( + process_rs.get(PROCESS_PID).unwrap().clone(), + ); + if pid == PID_NOT_FOUND { + return Ok(process_rs); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + check_pid(process_info_cmd, ssh_session, parse_process_pid).await + } + + fn monitor_command_name(command: &MonitorCommand) -> &'static str { + match command { + MonitorCommand::Start { .. } => "start", + MonitorCommand::Stop { .. } => "stop", + MonitorCommand::Restart { .. } => "restart", + MonitorCommand::Status { .. } => "status", + MonitorCommand::Update { .. } => "update", + } + } + pub fn new( config: DeployConfig, task_id: TaskId, @@ -208,9 +380,11 @@ impl MonitorCtlTask { let home = format!("{}/{}", config.install_dir(), GRAFANA_FILE_KEY); let cmd = MonitorComponentCommand::Grafana { home }; let task_name = match &cmd_arg { - SubCommand::Monitor { command, .. } => { - format!("grafana-{command}-{}", component_obj.port) - } + SubCommand::Monitor { command, .. } => format!( + "grafana-{}-{}", + Self::monitor_command_name(command), + component_obj.port + ), _ => unreachable!(), }; MonitorCtlTask::basic_component_ctl_task( @@ -238,9 +412,11 @@ impl MonitorCtlTask { let home = format!("{}/{}", config.install_dir(), PROMETHEUS_FILE_KEY); let cmd = MonitorComponentCommand::Prometheus { home }; let task_name = match &cmd_arg { - SubCommand::Monitor { command, .. } => { - format!("prometheus-{command}-{}", component_obj.port) - } + SubCommand::Monitor { command, .. } => format!( + "prometheus-{}-{}", + Self::monitor_command_name(command), + component_obj.port + ), _ => unreachable!(), }; MonitorCtlTask::basic_component_ctl_task( @@ -256,11 +432,43 @@ impl MonitorCtlTask { } } + pub fn alertmanager_ctl_task( + cmd_arg: SubCommand, + config: &DeployConfig, + ) -> IndexMap { + let monitor = config.deployment.monitor.as_ref().unwrap(); + if let Some(component_obj) = &monitor.alertmanager { + let component_host = component_obj.host.clone(); + let home = format!("{}/{}", config.install_dir(), ALERTMANAGER_FILE_KEY); + let cmd = MonitorComponentCommand::Alertmanager { home }; + let task_name = match &cmd_arg { + SubCommand::Monitor { command, .. } => format!( + "alertmanager-{}-{}", + Self::monitor_command_name(command), + component_obj.port + ), + _ => unreachable!(), + }; + MonitorCtlTask::basic_component_ctl_task( + cmd_arg, + config, + cmd, + component_host, + task_name, + ) + } else { + IndexMap::new() + } + } + pub fn exporter_ctl_task( cmd_arg: SubCommand, config: &DeployConfig, ) -> IndexMap { - let cmd_str_ref = cmd_arg.as_ref(); + let cmd_str_ref = match &cmd_arg { + SubCommand::Monitor { command, .. } => Self::monitor_command_name(command), + _ => unreachable!(), + }; let monitor = config.deployment.monitor.as_ref().unwrap(); let install_dir = config.install_dir(); let all_hosts = config.get_host_as_map(); @@ -300,6 +508,35 @@ impl MonitorCtlTask { .flatten() .collect::>() } + + pub fn prometheusalert_ctl_task( + cmd_arg: SubCommand, + config: &DeployConfig, + ) -> IndexMap { + let monitor = config.deployment.monitor.as_ref().unwrap(); + if let Some(component_obj) = &monitor.alertmanager { + let component_host = component_obj.host.clone(); + let home = format!("{}/{}", config.install_dir(), PROMETHEUSALERT_FILE_KEY); + let cmd = MonitorComponentCommand::PrometheusAlert { home }; + let task_name = match &cmd_arg { + SubCommand::Monitor { command, .. } => format!( + "alertmanager-webhook-adapter-{}-{}", + Self::monitor_command_name(command), + component_obj.webhook_adapter_port + ), + _ => unreachable!(), + }; + MonitorCtlTask::basic_component_ctl_task( + cmd_arg, + config, + cmd, + component_host, + task_name, + ) + } else { + IndexMap::new() + } + } } #[async_trait::async_trait] @@ -319,10 +556,7 @@ impl TaskExecutor for MonitorCtlTask { .await?; let cmd_str = match &self.cmd_args { - SubCommand::Monitor { - cluster: _, - command, - } => command.to_lowercase(), + SubCommand::Monitor { command, .. } => Self::monitor_command_name(command).to_string(), _ => unreachable!(), }; let process_info_cmd = self.monitor_ctl.process_info(); @@ -341,12 +575,24 @@ impl TaskExecutor for MonitorCtlTask { if monitor_component_pid.eq(PID_NOT_FOUND) { if let Some(start_cmd) = self.monitor_ctl.start(monitor_ref) { debug!(r#"MonitorCtlTask start_cmd={start_cmd:?}"#); - wait_command_complete!( - start_cmd.clone(), - process_info_cmd, - ssh_session.clone(), - is_some - ) + ssh_session + .command(start_cmd.as_str(), CollectOutput) + .await?; + let started = + Self::wait_started(ssh_session.clone(), process_info_cmd).await?; + let pid = TaskArgValue::into_inner_value::( + started.get(PROCESS_PID).unwrap().clone(), + ); + if pid == PID_NOT_FOUND { + let details = self.start_failure_details(ssh_session.clone()).await; + Err(anyhow::anyhow!( + "monitor component {} failed to start\n{}", + self.task_id.format_string(), + details + )) + } else { + Ok(started) + } } else { Ok(process_rs) } @@ -377,13 +623,36 @@ impl TaskExecutor for MonitorCtlTask { } "stop" => { if !monitor_component_pid.eq(PID_NOT_FOUND) { - let stop_cmd = self.monitor_ctl.stop(monitor_component_pid); - wait_command_complete!( - stop_cmd.clone(), - process_info_cmd, - ssh_session.clone(), - is_none - ) + let stop_cmd = self.monitor_ctl.stop(monitor_component_pid.clone()); + ssh_session + .command(stop_cmd.as_str(), CollectOutput) + .await?; + let stop_rs = + Self::wait_stopped(ssh_session.clone(), process_info_cmd.clone()).await?; + let pid = TaskArgValue::into_inner_value::( + stop_rs.get(PROCESS_PID).unwrap().clone(), + ); + if pid == PID_NOT_FOUND { + Ok(stop_rs) + } else { + let force_stop_cmd = self.monitor_ctl.force_stop(pid); + ssh_session + .command(force_stop_cmd.as_str(), CollectOutput) + .await?; + let force_stop_rs = + Self::wait_stopped(ssh_session.clone(), process_info_cmd).await?; + let force_pid = TaskArgValue::into_inner_value::( + force_stop_rs.get(PROCESS_PID).unwrap().clone(), + ); + if force_pid == PID_NOT_FOUND { + Ok(force_stop_rs) + } else { + Err(anyhow::anyhow!( + "monitor component {} failed to stop after SIGTERM and SIGKILL", + self.task_id.format_string() + )) + } + } } else { Ok(process_rs) } @@ -399,11 +668,7 @@ impl TaskExecutor for MonitorCtlTask { let pid = TaskArgValue::into_inner_value::( ctl_rtn_value.get(PROCESS_PID).unwrap().clone(), ); - let service_name = match &self.monitor_ctl { - MonitorComponentCommand::NodeExporter { .. } => "node_exporter", - MonitorComponentCommand::Prometheus { .. } => "prometheus", - MonitorComponentCommand::Grafana { .. } => "grafana", - }; + let service_name = self.monitor_ctl.service_name(); let output = if pid == PID_NOT_FOUND { format!("\n{service_name} is down.") } else { diff --git a/src/cluster_mgr/src/cli/task/redis_op_task.rs b/src/cluster_mgr/src/cli/task/redis_op_task.rs index e5bd3b8f..23ef6141 100644 --- a/src/cluster_mgr/src/cli/task/redis_op_task.rs +++ b/src/cluster_mgr/src/cli/task/redis_op_task.rs @@ -9,7 +9,7 @@ use anyhow::{anyhow, Error, Result}; use async_trait::async_trait; use futures::future::join_all; use redis::{Client, ErrorKind, RedisError, RedisResult, Value}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt; use std::hash::{Hash, Hasher}; @@ -28,6 +28,7 @@ pub struct RedisOpTask { password: Option, skip_checkpoint: bool, service_endpoints: Option>, + topology_retries: usize, } impl RedisOpTask { @@ -47,6 +48,7 @@ impl RedisOpTask { password, skip_checkpoint, service_endpoints: None, + topology_retries: TOPOLOGY_RETRIES, } } @@ -58,6 +60,11 @@ impl RedisOpTask { self } + pub fn with_topology_retries(mut self, topology_retries: usize) -> Self { + self.topology_retries = topology_retries.max(1); + self + } + // Helper method to check if the sender has any receivers fn has_receivers(&self) -> bool { self.sender.receiver_count() > 0 @@ -163,10 +170,11 @@ mod tests { } } -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct NodeInfo { pub ip: String, pub port: u16, + pub connected: bool, } // Implement Eq and Hash for NodeInfo @@ -192,7 +200,7 @@ impl fmt::Display for NodeInfo { } } -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct ClusterNodes { pub masters: Vec, pub replicas: Vec, @@ -329,7 +337,12 @@ pub fn parse_cluster_nodes(value: Value) -> RedisResult> { let is_master = !flags.contains("slave"); // Add node to appropriate list - let node_info = NodeInfo { ip, port }; + let connected = parts[7].eq_ignore_ascii_case("connected"); + let node_info = NodeInfo { + ip, + port, + connected, + }; if is_master { masters.push(node_info); @@ -385,7 +398,11 @@ pub fn parse_cluster_nodes_single(value: Value, default_host: &str) -> RedisResu }; Ok(ClusterNodes { - masters: vec![NodeInfo { ip, port }], + masters: vec![NodeInfo { + ip, + port, + connected: true, + }], replicas: vec![], }) } @@ -509,7 +526,7 @@ impl TaskExecutor for RedisOpTask { ErrorKind::IoError, "cluster topology query did not run", ))); - for attempt in 1..=TOPOLOGY_RETRIES { + for attempt in 1..=self.topology_retries { for node_url in &nodes { let client = match Client::open(node_url.clone()) { Ok(client) => client, @@ -540,10 +557,13 @@ impl TaskExecutor for RedisOpTask { break; } info!( - "RedisOpTask: cluster topology query failed, retrying {attempt}/{TOPOLOGY_RETRIES}. Attempted nodes: [{}]. Last error: {:?}", + "RedisOpTask: cluster topology query failed, retrying {attempt}/{}. Attempted nodes: [{}]. Last error: {:?}", + self.topology_retries, nodes_info, last_error ); - sleep(TOPOLOGY_RETRY_DELAY).await; + if attempt < self.topology_retries { + sleep(TOPOLOGY_RETRY_DELAY).await; + } } query_result } diff --git a/src/cluster_mgr/src/cli/task/rolling_upgrade/mod.rs b/src/cluster_mgr/src/cli/task/rolling_upgrade/mod.rs index 83bd3ad5..1a11f8b6 100644 --- a/src/cluster_mgr/src/cli/task/rolling_upgrade/mod.rs +++ b/src/cluster_mgr/src/cli/task/rolling_upgrade/mod.rs @@ -136,6 +136,7 @@ fn friendly_step_name(name: &str) -> &str { "StopTxNodes" => "Move traffic away from the current master", "StartTx" => "Start the original master node", "WaitCurrentMaster" => "Wait for a serving master", + "WaitTxReplicaReady" => "Wait for the original master to rejoin as replica", "FailoverBackAndStopStandby" => "Move traffic back and restart standby", "StartStandby" => "Start standby nodes", "StopVoters" => "Stop voter nodes", diff --git a/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs b/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs index d1145f5c..7e840878 100644 --- a/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs +++ b/src/cluster_mgr/src/cli/task/rolling_upgrade/steps.rs @@ -11,6 +11,7 @@ use crate::cli::task::local_extract_task::LocalExtractTask; use crate::cli::task::redis_op_task::{ClusterNodes, RedisOpTask}; use crate::cli::task::task_base::{TaskExecutionContext, TaskHost, TaskId, TaskInstance}; use crate::cli::task::upload::upload_task_builder::{upload_tasks, UploadTaskBuilderType}; +use crate::cli::task::wait_replica_ready_task::WaitReplicaReadyTask; use crate::cli::SubCommand; use crate::config::config_base::DeployConfig; use crate::config::storage_service_config::DataStoreServiceBackend; @@ -578,6 +579,83 @@ impl FailoverBackAndStopStandby { } } +pub struct WaitTxReplicaReady { + ctx: UpgradeContext, +} + +impl WaitTxReplicaReady { + pub fn new(ctx: UpgradeContext) -> Self { + Self { ctx } + } +} + +#[async_trait] +impl Step for WaitTxReplicaReady { + fn name(&self) -> &str { + "WaitTxReplicaReady" + } + + async fn build(&self) -> anyhow::Result { + if !self.ctx.has_standby() { + return Ok(TaskExecutionContext::dummy()); + } + + let tx_nodes = self.ctx.tx_host_ports(); + let standby_nodes = self.ctx.standby_host_ports(); + if tx_nodes.len() != standby_nodes.len() { + bail!( + "tx/standby node count mismatch: tx={}, standby={}", + tx_nodes.len(), + standby_nodes.len() + ); + } + let mut executable = IndexMap::new(); + + for (source_addr, target_addr) in standby_nodes.iter().zip(tx_nodes.iter()) { + let Some((source_host, source_port_str)) = source_addr.split_once(':') else { + bail!("invalid host:port in standby list: '{source_addr}'"); + }; + let Ok(source_port) = source_port_str.parse::() else { + bail!("invalid port in standby list: '{source_addr}'"); + }; + let Some((target_host, target_port_str)) = target_addr.split_once(':') else { + bail!("invalid host:port in tx list: '{target_addr}'"); + }; + let Ok(target_port) = target_port_str.parse::() else { + bail!("invalid port in tx list: '{target_addr}'"); + }; + let task_id = TaskId { + cmd: "topology".to_string(), + task: format!("wait-tx-replica-ready-{target_port}"), + host: target_host.to_string(), + }; + executable.insert( + task_id.clone(), + TaskInstance { + task_input: HashMap::default(), + task: Box::new( + WaitReplicaReadyTask::new( + task_id, + self.ctx.redis_cluster_startup_nodes(), + source_host.to_string(), + source_port, + target_host.to_string(), + target_port, + self.ctx.redis_password.clone(), + ) + .with_service_endpoints( + self.ctx.deploy.connection.service_endpoints.clone(), + ), + ), + task_host: TaskHost::Local, + }, + ); + } + + Ok(single_barrier_ctx("wait-tx-replica-ready", executable)) + } +} + #[async_trait] impl Step for FailoverBackAndStopStandby { fn name(&self) -> &str { @@ -769,6 +847,7 @@ pub fn build_upgrade_steps(ctx: UpgradeContext) -> Vec> { Box::new(StartLogAndWait::new(ctx.clone())), Box::new(StartTx::new(ctx.clone())), Box::new(WaitCurrentMaster::new(ctx.clone())), + Box::new(WaitTxReplicaReady::new(ctx.clone())), Box::new(FailoverBackAndStopStandby::new(ctx.clone())), Box::new(UnpackStandby::new(ctx.clone())), Box::new(StartStandby::new(ctx.clone())), @@ -784,6 +863,7 @@ pub fn build_config_restart_steps(ctx: UpgradeContext) -> Vec> { Box::new(StopTxNodes::new(ctx.clone())), Box::new(StartTx::new(ctx.clone())), Box::new(WaitCurrentMaster::new(ctx.clone())), + Box::new(WaitTxReplicaReady::new(ctx.clone())), Box::new(FailoverBackAndStopStandby::new(ctx.clone())), Box::new(StartStandby::new(ctx.clone())), Box::new(StopVoters::new(ctx.clone())), diff --git a/src/cluster_mgr/src/cli/task/task_base.rs b/src/cluster_mgr/src/cli/task/task_base.rs index bdc3be68..60bacbb2 100644 --- a/src/cluster_mgr/src/cli/task/task_base.rs +++ b/src/cluster_mgr/src/cli/task/task_base.rs @@ -1,5 +1,5 @@ use crate::cli::task::group::{init_task_group, Config}; -use crate::cli::task::task_controller::TaskController; +use crate::cli::task::task_controller::{summarize_error, task_action_summary, TaskController}; use crate::cli::{SubCommand, CMD, CMD_OUTPUT, CMD_STATUS}; use crate::config::connection::Connection; use crate::config::load_remote_env; @@ -156,6 +156,13 @@ pub fn merge_execution( ); tasks.executable.iter().for_each(|(id, _)| { debug!("add task {}", id.format_string()); + if executable.contains_key(id) { + tracing::warn!( + "duplicate task id detected while merging step {}: {}", + tasks.task_group, + id.format_string() + ); + } }); if !tasks.executable.is_empty() { @@ -288,6 +295,26 @@ impl TaskId { format!("host={},cmd={},task={}", self.host, self.cmd, self.task) } + pub fn parse_format_string(task_id: &str) -> Option { + let mut host = None; + let mut cmd = None; + let mut task = None; + for part in task_id.split(',') { + let (key, value) = part.split_once('=')?; + match key { + "host" => host = Some(value.to_string()), + "cmd" => cmd = Some(value.to_string()), + "task" => task = Some(value.to_string()), + _ => {} + } + } + Some(Self { + host: host?, + cmd: cmd?, + task: task?, + }) + } + pub fn as_json_string(&self) -> String { let task_id_string = serde_json::to_string(self); task_id_string.unwrap() @@ -427,8 +454,19 @@ impl TaskMgr { } TaskResultEnum::Success(None) => info!("task {task_id} finished"), TaskResultEnum::Error(err_msg) => { - let s = - format!("=> {task_id}\nFailure; {err_msg}\n---------------------------\n"); + let parsed = TaskId::parse_format_string(&task_id); + let action = parsed + .as_ref() + .map(|id| task_action_summary(&id.cmd, &id.task)) + .unwrap_or_else(|| "run task".to_string()); + let host = parsed + .as_ref() + .map(|id| id.host.clone()) + .unwrap_or_else(|| "unknown".to_string()); + let s = format!( + "=> {task_id}\nAction: {action}\nHost: {host}\nFailure: {}\n---------------------------\n", + summarize_error(&err_msg) + ); match writer { Some(ref mut w) => w.write_all(s.as_bytes())?, None => eprintln!("{s}"), diff --git a/src/cluster_mgr/src/cli/task/task_controller.rs b/src/cluster_mgr/src/cli/task/task_controller.rs index a930043e..d2d46cdf 100644 --- a/src/cluster_mgr/src/cli/task/task_controller.rs +++ b/src/cluster_mgr/src/cli/task/task_controller.rs @@ -1,6 +1,6 @@ use crate::cli::task::group::Config; use crate::cli::task::task_base::{ - init_finish_signal, is_verbose_task_output, TaskExecutionContext, + init_finish_signal, is_verbose_task_output, TaskExecutionContext, TaskId, }; use crate::cli::task::task_base::{TaskInstance, TaskResultEnum, TaskResultPair}; use crate::post_task_execute; @@ -90,7 +90,36 @@ impl TaskController { return "Refresh cluster topology".to_string(); } if task_ids.iter().all(|id| id.cmd == "monitor") { - return "Start monitor components".to_string(); + if task_ids.iter().all(|id| id.task.contains("-stop-")) { + return "Stop monitor components".to_string(); + } + if task_ids.iter().all(|id| id.task.contains("-status-")) { + return "Check monitor status".to_string(); + } + if task_ids.iter().all(|id| id.task.contains("-start-")) { + return "Start monitor components".to_string(); + } + return "Manage monitor components".to_string(); + } + if task_ids + .iter() + .all(|id| id.cmd == "remove" && id.task.starts_with("clean_log@")) + { + return "Delete log service data".to_string(); + } + if task_ids + .iter() + .all(|id| id.cmd == "remove" && id.task.starts_with("clean@")) + { + return "Delete cluster install directories".to_string(); + } + if task_ids.iter().all(|id| { + id.cmd == "backup" + && (id.task.starts_with("delete-") + || id.task.starts_with("remove-local-") + || id.task.starts_with("clean-backup")) + }) { + return "Delete backup data".to_string(); } Self::stage_label(task_group) @@ -215,7 +244,7 @@ impl TaskController { if is_verbose_task_output() { println!(" FAIL {} -- {}", task_id.format_string(), err_msg_str); } else { - println!(" failed: {}", user_friendly_task_label(&task_id.task)); + println!(" failed: {}", user_friendly_task_summary(&task_id)); } TaskResultEnum::Error(err_msg_str) } @@ -266,8 +295,8 @@ impl TaskController { fn stage_summary(task_count: usize) -> String { match task_count { 0 => "no work".to_string(), - 1 => "1 task".to_string(), - n => format!("{n} tasks"), + 1 => "1 operation".to_string(), + n => format!("{n} operations"), } } @@ -315,9 +344,9 @@ impl TaskController { for pair in rs.iter() { if let TaskResultEnum::Error(err) = &pair.result { if !is_verbose_task_output() { - println!(" failed: {err}"); + println!(" reason: {}", summarize_error(err)); } - bail!("task failed: {err}"); + bail!("operation failed: {err}"); } } } @@ -362,3 +391,60 @@ fn user_friendly_task_label(task: &str) -> String { .unwrap_or(task); humanize_task_name(task) } + +pub(crate) fn task_action_summary(cmd: &str, task: &str) -> String { + if cmd == "remove" && task.starts_with("clean@") { + return "remove cluster files".to_string(); + } + if cmd == "remove" && task.starts_with("clean_log@") { + return "remove log service files".to_string(); + } + if cmd == "monitor" { + if task.contains("grafana-stop") { + return "stop Grafana".to_string(); + } + if task.contains("grafana-start") { + return "start Grafana".to_string(); + } + if task.contains("grafana-status") { + return "check Grafana status".to_string(); + } + if task.contains("prometheus-stop") { + return "stop Prometheus".to_string(); + } + if task.contains("prometheus-start") { + return "start Prometheus".to_string(); + } + if task.contains("prometheus-status") { + return "check Prometheus status".to_string(); + } + if task.contains("node_exporter-stop") { + return "stop node_exporter".to_string(); + } + if task.contains("node_exporter-start") { + return "start node_exporter".to_string(); + } + if task.contains("node_exporter-status") { + return "check node_exporter status".to_string(); + } + } + user_friendly_task_label(task).to_lowercase() +} + +pub(crate) fn user_friendly_task_summary(task_id: &TaskId) -> String { + let action = task_action_summary(&task_id.cmd, &task_id.task); + if task_id.host.is_empty() || task_id.host == "local" || task_id.host == "127.0.0.1" { + action + } else { + format!("{action} on {}", task_id.host) + } +} + +pub(crate) fn summarize_error(err: &str) -> String { + if let Some(rest) = err.strip_prefix("Remote cmd [") { + if let Some((cmd, suffix)) = rest.split_once("] execution failed. status_code=") { + return format!("remote command exited with status {suffix}: {cmd}"); + } + } + err.to_string() +} diff --git a/src/cluster_mgr/src/cli/task/task_utils.rs b/src/cluster_mgr/src/cli/task/task_utils.rs index de395893..98a21cff 100644 --- a/src/cluster_mgr/src/cli/task/task_utils.rs +++ b/src/cluster_mgr/src/cli/task/task_utils.rs @@ -122,6 +122,7 @@ where // grep returns 1 when no matches found — this is normal, not an error. if status_code == 1 { let mut no_pid_result = cmd_exec_rs.clone(); + no_pid_result.insert(CMD_STATUS.to_string(), TaskArgValue::Number(0)); no_pid_result.insert( PROCESS_PID.to_string(), TaskArgValue::Str(PID_NOT_FOUND.to_string()), @@ -371,7 +372,8 @@ pub async fn stop_with_hot_standby( redis_op_password, skip_checkpoint, ) - .with_service_endpoints(config.connection.service_endpoints.clone()); + .with_service_endpoints(config.connection.service_endpoints.clone()) + .with_topology_retries(1); let task_instance = TaskInstance { task_input: HashMap::default(), @@ -460,7 +462,8 @@ pub async fn stop_with_failover( redis_op_password.clone(), skip_checkpoint, ) - .with_service_endpoints(config.connection.service_endpoints.clone()); + .with_service_endpoints(config.connection.service_endpoints.clone()) + .with_topology_retries(1); let topology_instance = TaskInstance { task_input: HashMap::default(), diff --git a/src/cluster_mgr/src/cli/task/unpack_file_task.rs b/src/cluster_mgr/src/cli/task/unpack_file_task.rs index df078ffd..2dfca28d 100644 --- a/src/cluster_mgr/src/cli/task/unpack_file_task.rs +++ b/src/cluster_mgr/src/cli/task/unpack_file_task.rs @@ -11,9 +11,11 @@ use itertools::Itertools; use std::collections::HashMap; use tracing::info; -pub(crate) const REMOTE_UNPACKED_NAMES: [&str; 6] = [ +pub(crate) const REMOTE_UNPACKED_NAMES: [&str; 8] = [ "prometheus", + "alertmanager", "grafana", + "PrometheusAlert", "node_exporter", "eloq-logserver", "eloqkv-proxy", @@ -33,6 +35,9 @@ fn extract_unpacked_name(raw_file_name: &str) -> String { if raw_file_name.starts_with("eloqdb") { return "eloqdb".to_string(); } + if raw_file_name.contains("PrometheusAlert") || raw_file_name == "linux.zip" { + return "alertmanager-webhook-adapter".to_string(); + } for unpacked in REMOTE_UNPACKED_NAMES { if !raw_file_name.contains(unpacked) { continue; diff --git a/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs b/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs index 544ec3ac..ea5b6faa 100644 --- a/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/eloq_upload_builder.rs @@ -6,8 +6,8 @@ use crate::cli::task::upload::upload_task_builder::{ }; use crate::cli::{create_upload_cluster_dir, upload_dir}; use crate::config::config_base::{ - DeployConfig, UploadFile, ELOQ_FILE_KEY, ELOQ_LOG_FILE_KEY, GRAFANA_FILE_KEY, - NODE_EXPORTER_FILE_KEY, PROMETHEUS_FILE_KEY, + DeployConfig, UploadFile, ALERTMANAGER_FILE_KEY, ELOQ_FILE_KEY, ELOQ_LOG_FILE_KEY, + GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, PROMETHEUSALERT_FILE_KEY, PROMETHEUS_FILE_KEY, }; use crate::config::deployment::Deployment; use crate::config::storage_service_config::RocksDB; @@ -53,7 +53,11 @@ impl EloqUploadBuilder { .concat(), ELOQ_LOG_FILE_KEY => log_hosts.clone(), PROMETHEUS_FILE_KEY => config.get_host_list(DeploymentPackage::Prometheus), + ALERTMANAGER_FILE_KEY => config.get_host_list(DeploymentPackage::Alertmanager), GRAFANA_FILE_KEY => config.get_host_list(DeploymentPackage::Grafana), + PROMETHEUSALERT_FILE_KEY => { + config.get_host_list(DeploymentPackage::PrometheusAlert) + } _ => unreachable!(), }; (dest_hosts, download_url, file_key.clone()) @@ -86,7 +90,9 @@ impl EloqUploadBuilder { ELOQ_FILE_KEY => config.deployment.product().home().to_string(), ELOQ_LOG_FILE_KEY => "LogServer".to_string(), PROMETHEUS_FILE_KEY => "prometheus".to_string(), + ALERTMANAGER_FILE_KEY => "alertmanager".to_string(), GRAFANA_FILE_KEY => "grafana".to_string(), + PROMETHEUSALERT_FILE_KEY => PROMETHEUSALERT_FILE_KEY.to_string(), NODE_EXPORTER_FILE_KEY => "node_exporter".to_string(), _ => unreachable!(), } diff --git a/src/cluster_mgr/src/cli/task/upload/mod.rs b/src/cluster_mgr/src/cli/task/upload/mod.rs index e24b29c2..c0c41147 100644 --- a/src/cluster_mgr/src/cli/task/upload/mod.rs +++ b/src/cluster_mgr/src/cli/task/upload/mod.rs @@ -1,6 +1,6 @@ mod data_dir_upload_builder; mod eloq_upload_builder; -mod monitor_upload_builder; +pub mod monitor_upload_builder; mod proxy_upload_builder; mod tx_conf_upload_builder; mod upload_task; diff --git a/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs b/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs index a5610e0b..3f1ecf44 100644 --- a/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs +++ b/src/cluster_mgr/src/cli/task/upload/monitor_upload_builder.rs @@ -6,18 +6,80 @@ use crate::cli::task::upload::upload_task_builder::{ use crate::cli::upload_dir; use crate::config::config_base::{DeployConfig, UploadFile}; use crate::config::monitor::{ - Monitor, GRAFANA_CONFIG_DIR, GRAFANA_DASHBOARD_CONFIG_DIR, GRAFANA_DATASOURCE_CONFIG_DIR, + Monitor, ALERTMANAGER_CONFIG_DIR, ALERTMANAGER_WEBHOOK_ADAPTER_TEMPLATE_DIR, + GRAFANA_CONFIG_DIR, GRAFANA_DASHBOARD_CONFIG_DIR, GRAFANA_DATASOURCE_CONFIG_DIR, MONITOR_JOB_NAME, NODE_EXPORTER_JOB_NAME, PROMETHEUS_CONFIG_DIR, }; use crate::config::{DeploymentPackage, ALERT_RULES_TEMPLATE, MONITOR_DIR}; +use anyhow::anyhow; use indexmap::IndexMap; -use itertools::Itertools; use std::collections::HashMap; +use std::fs; +use std::path::Path; use std::path::PathBuf; pub struct MonitorInfraConfUploadBuilder; impl MonitorInfraConfUploadBuilder { + fn stage_named_dir( + &self, + config: &DeployConfig, + dir_name: &str, + files: &[PathBuf], + ) -> anyhow::Result { + let staged_dir = upload_dir() + .join(&config.deployment.cluster_name) + .join(MONITOR_DIR) + .join(dir_name); + if staged_dir.exists() { + fs::remove_dir_all(&staged_dir)?; + } + fs::create_dir_all(&staged_dir)?; + + for source_path in files { + let file_name = source_path + .file_name() + .ok_or_else(|| anyhow!("staged source path has no file name"))?; + fs::copy(source_path, staged_dir.join(file_name))?; + } + + Ok(staged_dir) + } + + fn stage_dashboard_upload_dir( + &self, + config: &DeployConfig, + dashboard_files: &[String], + dashboard_config_path: &Path, + ) -> anyhow::Result { + let staged_dir = upload_dir() + .join(&config.deployment.cluster_name) + .join(MONITOR_DIR) + .join("grafana_dashboards"); + if staged_dir.exists() { + fs::remove_dir_all(&staged_dir)?; + } + fs::create_dir_all(&staged_dir)?; + + for source in dashboard_files { + let source_path = PathBuf::from(source); + let file_name = source_path + .file_name() + .ok_or_else(|| anyhow!("dashboard source path has no file name: {source}"))?; + fs::copy(&source_path, staged_dir.join(file_name))?; + } + + let dashboard_config_name = dashboard_config_path + .file_name() + .ok_or_else(|| anyhow!("dashboard config path has no file name"))?; + fs::copy( + dashboard_config_path, + staged_dir.join(dashboard_config_name), + )?; + + Ok(staged_dir) + } + fn dashboard_upload_files(&self, config: &DeployConfig) -> Option { let files = config.load_monitor_dashboard(); let install_dir = config.install_dir(); @@ -31,24 +93,22 @@ impl MonitorInfraConfUploadBuilder { ); assert!(dashboard_path.is_ok()); let path_binding = dashboard_path.unwrap(); - let local_config_path_string = path_binding.to_str().unwrap().to_string(); - if files.is_empty() || monitor.grafana.is_none() { None } else { let host = config.get_host_list(DeploymentPackage::Grafana); assert_eq!(1, host.len()); let dest_host = host.first().unwrap(); - let mut dashboard_files = files.iter().join(" "); - dashboard_files.push(' '); - dashboard_files.push_str(local_config_path_string.as_str()); + let staged_dashboard_dir = self + .stage_dashboard_upload_dir(config, &files, path_binding.as_path()) + .expect("stage grafana dashboard upload dir"); let install_dir = config.install_dir(); Some(UploadFile { - source: dashboard_files, + source: staged_dashboard_dir.to_string_lossy().to_string(), dest: format!("{install_dir}/{GRAFANA_DASHBOARD_CONFIG_DIR}"), - extension: "json".to_string(), + extension: "dashboard_dir".to_string(), host: dest_host.to_string(), - copy_dir: false, + copy_dir: true, delete_remote: false, }) } @@ -75,7 +135,10 @@ impl MonitorInfraConfUploadBuilder { config: &DeployConfig, ) -> Vec { // Return an empty vector if both Prometheus and Grafana are not configured - if monitor.prometheus.is_none() && monitor.grafana.is_none() { + if monitor.prometheus.is_none() + && monitor.alertmanager.is_none() + && monitor.grafana.is_none() + { return vec![]; } @@ -148,6 +211,46 @@ impl MonitorInfraConfUploadBuilder { upload_files.push(prometheus_upload_file); } + if monitor.alertmanager.is_some() { + let alertmanager_conf = monitor + .gen_alertmanager_config(&config.deployment.cluster_name) + .unwrap(); + let alertmanager_webhook_template = monitor + .gen_alertmanager_webhook_adapter_template(&config.deployment.cluster_name) + .unwrap(); + let staged_template_dir = self + .stage_named_dir( + config, + "alertmanager_webhook_adapter_templates", + &[alertmanager_webhook_template], + ) + .unwrap(); + let alertmanager_upload_file = UploadFile { + source: MonitorInfraConfUploadBuilder::path_string(alertmanager_conf), + dest: format!("{install_dir}/{ALERTMANAGER_CONFIG_DIR}"), + extension: "alertmanager".to_string(), + host: MonitorInfraConfUploadBuilder::monitor_host( + &all_host, + DeploymentPackage::Alertmanager, + ), + copy_dir: false, + delete_remote: false, + }; + upload_files.push(alertmanager_upload_file); + let alertmanager_webhook_template_upload_file = UploadFile { + source: staged_template_dir.to_string_lossy().to_string(), + dest: format!("{install_dir}/{ALERTMANAGER_WEBHOOK_ADAPTER_TEMPLATE_DIR}"), + extension: "alertmanager-webhook-adapter.tmpl".to_string(), + host: MonitorInfraConfUploadBuilder::monitor_host( + &all_host, + DeploymentPackage::Alertmanager, + ), + copy_dir: true, + delete_remote: false, + }; + upload_files.push(alertmanager_webhook_template_upload_file); + } + // Handle Grafana configuration separately if monitor.grafana.is_some() { let grafana_ds_conf_path = monitor @@ -195,6 +298,13 @@ impl MonitorInfraConfUploadBuilder { impl UploadTaskBuilder for MonitorInfraConfUploadBuilder { fn build(&self, config: &Config) -> IndexMap { + Self::build_for_cmd(config, "deploy") + } +} + +impl MonitorInfraConfUploadBuilder { + pub fn build_for_cmd(config: &Config, cmd: &str) -> IndexMap { + let builder = MonitorInfraConfUploadBuilder; let cluster_config = match config { Config::Cluster(cfg) => cfg, _ => panic!("Expected ClusterConfig for MonitorInfraConfUploadBuilder"), @@ -210,9 +320,10 @@ impl UploadTaskBuilder for MonitorInfraConfUploadBuilder { .expect("generate alert rules error"); } - let mut all_upload_files = self.monitor_config_upload_files(monitor, cluster_config); + let mut all_upload_files = builder.monitor_config_upload_files(monitor, cluster_config); if monitor.grafana.is_some() { - if let Some(upload_dashboard_file) = self.dashboard_upload_files(cluster_config) { + if let Some(upload_dashboard_file) = builder.dashboard_upload_files(cluster_config) + { all_upload_files.push(upload_dashboard_file); } } @@ -224,7 +335,7 @@ impl UploadTaskBuilder for MonitorInfraConfUploadBuilder { source_host.clone(), upload_file.clone(), config, - "deploy", + cmd, format!("upload_monitor_cnf_{extension}").as_str(), ) }) diff --git a/src/cluster_mgr/src/cli/task/wait_replica_ready_task.rs b/src/cluster_mgr/src/cli/task/wait_replica_ready_task.rs new file mode 100644 index 00000000..d8b55036 --- /dev/null +++ b/src/cluster_mgr/src/cli/task/wait_replica_ready_task.rs @@ -0,0 +1,221 @@ +use crate::cli::task::redis_op_task::{ClusterNodes, RedisOpTask}; +use crate::cli::task::task_base::{ExecutionValue, TaskArgValue, TaskExecutor, TaskHost, TaskId}; +use crate::cli::{CMD, CMD_OUTPUT, CMD_STATUS}; +use crate::config::connection::ServiceEndpoint; +use anyhow::Result; +use async_trait::async_trait; +use std::collections::HashMap; +use std::time::Duration; +use tokio::sync::watch; +use tokio::time::sleep; + +const REPLICA_READY_RETRIES: usize = 120; +const REPLICA_READY_RETRY_DELAY: Duration = Duration::from_secs(2); + +#[derive(Clone, Debug)] +pub struct WaitReplicaReadyTask { + task_id: TaskId, + startup_nodes: Vec, + source_host: String, + source_port: u16, + target_host: String, + target_port: u16, + password: Option, + service_endpoints: Option>, +} + +impl WaitReplicaReadyTask { + pub fn new( + task_id: TaskId, + startup_nodes: Vec, + source_host: String, + source_port: u16, + target_host: String, + target_port: u16, + password: Option, + ) -> Self { + Self { + task_id, + startup_nodes, + source_host, + source_port, + target_host, + target_port, + password, + service_endpoints: None, + } + } + + pub fn with_service_endpoints( + mut self, + service_endpoints: Option>, + ) -> Self { + self.service_endpoints = service_endpoints; + self + } + + fn find_connected_master<'a>( + &self, + cluster_nodes: &'a ClusterNodes, + ) -> Option<&'a crate::cli::task::redis_op_task::NodeInfo> { + cluster_nodes.masters.iter().find(|node| { + node.ip == self.source_host && node.port == self.source_port && node.connected + }) + } + + fn find_connected_target_replica<'a>( + &self, + cluster_nodes: &'a ClusterNodes, + ) -> Option<&'a crate::cli::task::redis_op_task::NodeInfo> { + cluster_nodes.replicas.iter().find(|node| { + node.ip == self.target_host && node.port == self.target_port && node.connected + }) + } + + async fn fetch_cluster_nodes(&self) -> Result { + let task_id = TaskId { + cmd: "topology".to_string(), + task: format!("{}-topology", self.task_id.task), + host: "_local".to_string(), + }; + let (tx, _rx) = watch::channel(ClusterNodes { + masters: Vec::new(), + replicas: Vec::new(), + }); + let result = RedisOpTask::new( + task_id, + self.startup_nodes.clone(), + "cluster topology".to_string(), + tx, + self.password.clone(), + true, + ) + .with_service_endpoints(self.service_endpoints.clone()) + .execute(TaskHost::Local, HashMap::default()) + .await?; + + let values = result.ok_or_else(|| anyhow::anyhow!("missing topology task result"))?; + let output = values + .get(CMD_OUTPUT) + .cloned() + .unwrap_or_else(|| TaskArgValue::Str("missing cluster topology output".to_string())); + let status = values + .get(CMD_STATUS) + .cloned() + .unwrap_or(TaskArgValue::Number(1)); + + match (status, output) { + (TaskArgValue::Number(0), TaskArgValue::Str(json)) => { + Ok(serde_json::from_str::(&json)?) + } + (_, TaskArgValue::Str(err)) => Err(anyhow::anyhow!(err)), + _ => Err(anyhow::anyhow!("unexpected topology task output")), + } + } +} + +#[async_trait] +impl TaskExecutor for WaitReplicaReadyTask { + fn identifier(&self) -> TaskId { + self.task_id.clone() + } + + async fn execute( + &self, + _task_host: TaskHost, + _task_arg: HashMap, + ) -> Result> { + let mut task_result = HashMap::from([( + CMD.to_string(), + TaskArgValue::Str("wait replica ready".to_string()), + )]); + + let source = format!("{}:{}", self.source_host, self.source_port); + let target = format!("{}:{}", self.target_host, self.target_port); + let mut last_seen = + String::from("required nodes not yet observed as connected in cluster topology"); + + for _ in 0..REPLICA_READY_RETRIES { + match self.fetch_cluster_nodes().await { + Ok(cluster_nodes) => { + let masters = cluster_nodes + .masters + .iter() + .map(|node| { + format!( + "{}:{} ({})", + node.ip, + node.port, + if node.connected { + "connected" + } else { + "disconnected" + } + ) + }) + .collect::>(); + let replicas = cluster_nodes + .replicas + .iter() + .map(|node| { + format!( + "{}:{} ({})", + node.ip, + node.port, + if node.connected { + "connected" + } else { + "disconnected" + } + ) + }) + .collect::>(); + if self.find_connected_master(&cluster_nodes).is_some() + && self.find_connected_target_replica(&cluster_nodes).is_some() + { + task_result.insert(CMD_STATUS.to_string(), TaskArgValue::Number(0)); + task_result.insert( + CMD_OUTPUT.to_string(), + TaskArgValue::Str(format!( + "Master {source} and replica {target} are connected and ready for failover. Masters: {}. Replicas: {}", + masters.join(", "), + replicas.join(", ") + )), + ); + return Ok(Some(task_result)); + } + last_seen = if masters.is_empty() && replicas.is_empty() { + "cluster currently reports no masters or replicas".to_string() + } else { + format!( + "masters currently visible: {}; replicas currently visible: {}", + if masters.is_empty() { + "".to_string() + } else { + masters.join(", ") + }, + if replicas.is_empty() { + "".to_string() + } else { + replicas.join(", ") + } + ) + }; + } + Err(err) => { + last_seen = err.to_string(); + } + } + sleep(REPLICA_READY_RETRY_DELAY).await; + } + + task_result.insert(CMD_STATUS.to_string(), TaskArgValue::Number(1)); + task_result.insert( + CMD_OUTPUT.to_string(), + TaskArgValue::Str(format!( + "Master {source} and replica {target} did not both become connected in time: {last_seen}" + )), + ); + Ok(Some(task_result)) + } +} diff --git a/src/cluster_mgr/src/config/config_base.rs b/src/cluster_mgr/src/config/config_base.rs index cde1e150..c532c55c 100644 --- a/src/cluster_mgr/src/config/config_base.rs +++ b/src/cluster_mgr/src/config/config_base.rs @@ -22,7 +22,9 @@ pub const LOG_SERVICE_HOME: &str = "LogServer"; pub const ELOQ_FILE_KEY: &str = "eloq_tx"; pub const ELOQ_LOG_FILE_KEY: &str = "eloq_log"; pub const PROMETHEUS_FILE_KEY: &str = "prometheus"; +pub const ALERTMANAGER_FILE_KEY: &str = "alertmanager"; pub const GRAFANA_FILE_KEY: &str = "grafana"; +pub const PROMETHEUSALERT_FILE_KEY: &str = "alertmanager-webhook-adapter"; pub const NODE_EXPORTER_FILE_KEY: &str = "node_exporter"; pub const DEPLOYMENT_CHECK_SUCCESS_TASK: &str = "deploy_check_success_task"; pub const SCALED_CLUSTER_CONFIG: &str = "cluster_config"; @@ -280,9 +282,15 @@ impl DeployConfig { DeploymentPackage::Prometheus => { extract_monitor_link!(monitor_link, PROMETHEUS_FILE_KEY, unpack_files); } + DeploymentPackage::Alertmanager => { + extract_monitor_link!(monitor_link, ALERTMANAGER_FILE_KEY, unpack_files); + } DeploymentPackage::Grafana => { extract_monitor_link!(monitor_link, GRAFANA_FILE_KEY, unpack_files); } + DeploymentPackage::PrometheusAlert => { + extract_monitor_link!(monitor_link, PROMETHEUSALERT_FILE_KEY, unpack_files); + } DeploymentPackage::Proxy => unreachable!(), } @@ -519,10 +527,18 @@ impl DeployConfig { DeploymentPackage::Prometheus, self.get_host_list(DeploymentPackage::Prometheus), ), + ( + DeploymentPackage::Alertmanager, + self.get_host_list(DeploymentPackage::Alertmanager), + ), ( DeploymentPackage::Grafana, self.get_host_list(DeploymentPackage::Grafana), ), + ( + DeploymentPackage::PrometheusAlert, + self.get_host_list(DeploymentPackage::PrometheusAlert), + ), ]) } @@ -534,8 +550,10 @@ impl DeployConfig { EloqVoter, EloqLog, Storage, + Alertmanager, Grafana, - Prometheus + Prometheus, + PrometheusAlert ); all_hosts.iter().unique().cloned().collect_vec() } diff --git a/src/cluster_mgr/src/config/deployment.rs b/src/cluster_mgr/src/config/deployment.rs index 66200df6..9dc100b4 100644 --- a/src/cluster_mgr/src/config/deployment.rs +++ b/src/cluster_mgr/src/config/deployment.rs @@ -2139,9 +2139,15 @@ impl Deployment { DeploymentPackage::Prometheus => { extract_monitor_host!(self, prometheus) } + DeploymentPackage::Alertmanager => { + extract_monitor_host!(self, alertmanager) + } DeploymentPackage::Grafana => { extract_monitor_host!(self, grafana) } + DeploymentPackage::PrometheusAlert => { + extract_monitor_host!(self, alertmanager) + } DeploymentPackage::Proxy => unreachable!(), } } @@ -2173,8 +2179,48 @@ impl Deployment { DeploymentPackage::EloqVoter => { self.get_host_port_list_internal(&self.tx_service.voter_host_ports) } - DeploymentPackage::Prometheus => vec![], - DeploymentPackage::Grafana => vec![], + DeploymentPackage::Prometheus => self + .monitor + .as_ref() + .and_then(|monitor| { + monitor + .prometheus + .as_ref() + .map(|component| vec![format!("{}:{}", component.host, component.port)]) + }) + .unwrap_or_default(), + DeploymentPackage::Alertmanager => self + .monitor + .as_ref() + .and_then(|monitor| { + monitor + .alertmanager + .as_ref() + .map(|component| vec![format!("{}:{}", component.host, component.port)]) + }) + .unwrap_or_default(), + DeploymentPackage::Grafana => self + .monitor + .as_ref() + .and_then(|monitor| { + monitor + .grafana + .as_ref() + .map(|component| vec![format!("{}:{}", component.host, component.port)]) + }) + .unwrap_or_default(), + DeploymentPackage::PrometheusAlert => self + .monitor + .as_ref() + .and_then(|monitor| { + monitor.alertmanager.as_ref().map(|component| { + vec![format!( + "{}:{}", + component.host, component.webhook_adapter_port + )] + }) + }) + .unwrap_or_default(), DeploymentPackage::Proxy => unreachable!(), } } @@ -2201,7 +2247,9 @@ impl Deployment { self.populate_topo(&mut topo, DeploymentPackage::EloqStandby); self.populate_topo(&mut topo, DeploymentPackage::EloqVoter); self.populate_topo(&mut topo, DeploymentPackage::Prometheus); + self.populate_topo(&mut topo, DeploymentPackage::Alertmanager); self.populate_topo(&mut topo, DeploymentPackage::Grafana); + self.populate_topo(&mut topo, DeploymentPackage::PrometheusAlert); self.populate_topo(&mut topo, DeploymentPackage::EloqLog); topo } diff --git a/src/cluster_mgr/src/config/mod.rs b/src/cluster_mgr/src/config/mod.rs index 44d406b4..854d4c70 100644 --- a/src/cluster_mgr/src/config/mod.rs +++ b/src/cluster_mgr/src/config/mod.rs @@ -29,16 +29,22 @@ pub const ELOQKV_NODE_INI: &str = "EloqKv-node"; pub const START_LOG_TEMPLATE: &str = "start_tx_log.bash"; pub const JVM_SETTING_HOLDER: &str = "_GC_SETTINGS_PLACEHOLDER_"; pub const PROMETHEUS_CONFIG_TEMPLATE: &str = "mono_prometheus.yaml"; +pub const ALERTMANAGER_CONFIG_TEMPLATE: &str = "mono_alertmanager.yaml"; pub const MONITOR_DIR: &str = "monitor"; pub const ALERT_RULES_TEMPLATE: &str = "alert.rules"; pub const GRAFANA_DASHBOARDS_CONFIG_TEMPLATE: &str = "grafana_dashboards.yaml"; pub const PROMETHEUS_CONFIG_FILE: &str = "prometheus.yml"; +pub const ALERTMANAGER_CONFIG_FILE: &str = "alertmanager.yml"; pub const GRAFANA_PROMETHEUS_DS_FILE: &str = "prometheus-datasource.yml"; -pub const GRAFANA_CONFIG_TEMPLATE: &str = "grafana_config.ini"; -pub const GRAFANA_CONFIG_FILE: &str = "defaults.ini"; +pub const GRAFANA_CONFIG_TEMPLATE_V9: &str = "grafana_config.ini"; +pub const GRAFANA_CONFIG_TEMPLATE_V13: &str = "grafana_config_v13.ini"; +pub const GRAFANA_CONFIG_FILE: &str = "custom.ini"; +pub const ALERTMANAGER_WEBHOOK_ADAPTER_FEISHU_TEMPLATE: &str = "feishu.zh.tmpl"; +pub const PROMETHEUSALERT_CONFIG_TEMPLATE: &str = "prometheusalert_app.conf"; +pub const PROMETHEUSALERT_CONFIG_FILE: &str = "app.conf"; pub const CDN: &str = "https://download.eloqdata.com"; @@ -176,8 +182,12 @@ pub enum DeploymentPackage { Storage, #[strum(serialize = "prometheus")] Prometheus, + #[strum(serialize = "alertmanager")] + Alertmanager, #[strum(serialize = "grafana")] Grafana, + #[strum(serialize = "prometheusalert")] + PrometheusAlert, #[strum(serialize = "eloq_log")] EloqLog, #[strum(serialize = "eloq_standby")] diff --git a/src/cluster_mgr/src/config/monitor.rs b/src/cluster_mgr/src/config/monitor.rs index 160c0426..22d6fe44 100644 --- a/src/cluster_mgr/src/config/monitor.rs +++ b/src/cluster_mgr/src/config/monitor.rs @@ -1,9 +1,14 @@ use crate::cli::upload_dir; -use crate::config::config_base::{GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, PROMETHEUS_FILE_KEY}; +use crate::config::config_base::{ + ALERTMANAGER_FILE_KEY, GRAFANA_FILE_KEY, NODE_EXPORTER_FILE_KEY, PROMETHEUSALERT_FILE_KEY, + PROMETHEUS_FILE_KEY, +}; use crate::config::{ - config_template, load_yaml_config_template, DownloadUrl, GRAFANA_CONFIG_FILE, - GRAFANA_CONFIG_TEMPLATE, GRAFANA_DASHBOARDS_CONFIG_TEMPLATE, GRAFANA_PROMETHEUS_DS_FILE, - MONITOR_DIR, PROMETHEUS_CONFIG_FILE, PROMETHEUS_CONFIG_TEMPLATE, + config_template, load_yaml_config_template, DownloadUrl, ALERTMANAGER_CONFIG_FILE, + ALERTMANAGER_CONFIG_TEMPLATE, ALERTMANAGER_WEBHOOK_ADAPTER_FEISHU_TEMPLATE, + GRAFANA_CONFIG_FILE, GRAFANA_CONFIG_TEMPLATE_V13, GRAFANA_CONFIG_TEMPLATE_V9, + GRAFANA_DASHBOARDS_CONFIG_TEMPLATE, GRAFANA_PROMETHEUS_DS_FILE, MONITOR_DIR, + PROMETHEUS_CONFIG_FILE, PROMETHEUS_CONFIG_TEMPLATE, }; use crate::download_urls; use itertools::Itertools; @@ -18,6 +23,14 @@ pub const GRAFANA_CONFIG_DIR: &str = "grafana/conf"; pub const GRAFANA_DATASOURCE_CONFIG_DIR: &str = "grafana/conf/provisioning/datasources"; pub const GRAFANA_DASHBOARD_CONFIG_DIR: &str = "grafana/conf/provisioning/dashboards"; pub const PROMETHEUS_CONFIG_DIR: &str = "prometheus"; +pub const ALERTMANAGER_CONFIG_DIR: &str = "alertmanager"; +pub const ALERTMANAGER_WEBHOOK_ADAPTER_DIR: &str = "alertmanager-webhook-adapter"; +pub const ALERTMANAGER_WEBHOOK_ADAPTER_BINARY: &str = "alertmanager-webhook-adapter"; +pub const ALERTMANAGER_WEBHOOK_ADAPTER_TEMPLATE_DIR: &str = + "alertmanager-webhook-adapter/templates"; +pub const ALERTMANAGER_WEBHOOK_ADAPTER_PORT: u16 = 18080; +pub const ALERTMANAGER_WEBHOOK_ADAPTER_DEFAULT_URL: &str = + "https://github.com/bougou/alertmanager-webhook-adapter/releases/download/v1.1.11/alertmanager-webhook-adapter-v1.1.11-linux-amd64"; pub const NODE_EXPORTER_JOB_NAME: &str = "eloq-node"; @@ -28,7 +41,9 @@ macro_rules! monitor_component_config_dir { ($component:expr) => {{ match $component.to_lowercase().as_str() { "prometheus" => "prometheus".to_string(), + "alertmanager" => "alertmanager".to_string(), "grafana" => "grafana/conf/provisioning/datasources".to_string(), + "prometheusalert" => "alertmanager-webhook-adapter".to_string(), _ => unreachable!(), } }}; @@ -48,6 +63,28 @@ macro_rules! monitor_components { monitor_components!(Grafana); +#[serde_with::skip_serializing_none] +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] +pub struct Alertmanager { + pub download_url: String, + pub port: u16, + pub host: String, + #[serde(default)] + pub feishu_robot_urls: Option>, + #[serde(default = "default_alertmanager_webhook_adapter_download_url")] + pub webhook_adapter_download_url: String, + #[serde(default = "default_alertmanager_webhook_adapter_port")] + pub webhook_adapter_port: u16, +} + +fn default_alertmanager_webhook_adapter_download_url() -> String { + ALERTMANAGER_WEBHOOK_ADAPTER_DEFAULT_URL.to_string() +} + +fn default_alertmanager_webhook_adapter_port() -> u16 { + ALERTMANAGER_WEBHOOK_ADAPTER_PORT +} + #[serde_with::skip_serializing_none] #[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] pub struct Prometheus { @@ -159,12 +196,36 @@ impl Default for AlertThresholds { pub struct Monitor { pub data_dir: Option, pub prometheus: Option, + pub alertmanager: Option, pub grafana: Option, pub node_exporter: Option, pub eloq_metrics: Option, } impl Monitor { + fn grafana_is_modern(download_url: &str) -> bool { + Self::grafana_major_version(download_url).unwrap_or(9) >= 13 + } + + fn grafana_major_version(download_url: &str) -> Option { + let filename = download_url + .rsplit('/') + .next() + .unwrap_or(download_url) + .trim_end_matches(".tar.gz"); + let mut digit_start = None; + for (idx, ch) in filename.char_indices() { + if ch.is_ascii_digit() { + digit_start = Some(idx); + break; + } + } + let start = digit_start?; + let version_tail = &filename[start..]; + let major = version_tail.split('.').next()?; + major.parse().ok() + } + pub fn download_links(&self) -> anyhow::Result> { let download_links = self.download_links_as_map()?; Ok(download_links.into_values().collect_vec()) @@ -175,6 +236,16 @@ impl Monitor { if let Some(prom) = &self.prometheus { download_urls!(links, {PROMETHEUS_FILE_KEY, &prom.download_url}); } + if let Some(alertmanager) = &self.alertmanager { + download_urls!(links, {ALERTMANAGER_FILE_KEY, &alertmanager.download_url}); + download_urls!( + links, + { + PROMETHEUSALERT_FILE_KEY, + &alertmanager.webhook_adapter_download_url + } + ); + } if let Some(graf) = &self.grafana { download_urls!(links, {GRAFANA_FILE_KEY, &graf.download_url}); } @@ -214,7 +285,12 @@ impl Monitor { pub fn gen_grafana_config(&self, cluster_name: &str) -> anyhow::Result { if let Some(graf) = &self.grafana { let grafana_http_port = graf.port; - let grafana_config_path = config_template(GRAFANA_CONFIG_TEMPLATE)?; + let template = if Self::grafana_is_modern(&graf.download_url) { + GRAFANA_CONFIG_TEMPLATE_V13 + } else { + GRAFANA_CONFIG_TEMPLATE_V9 + }; + let grafana_config_path = config_template(template)?; let mut grafana_ini = ini::Ini::load_from_file(grafana_config_path) .expect("can not load grafana config template"); grafana_ini.set_to( @@ -346,6 +422,7 @@ impl Monitor { "${MAX_STANDBY_LAG_PCT_DECIMAL}", &format!("{:.1}", threshold.max_standby_lag_pct as f64 / 100.0), ); + template = template.replace("${CLUSTER_NAME}", cluster_name); let output_dir = upload_dir().join(cluster_name).join(MONITOR_DIR); fs::create_dir_all(&output_dir)?; @@ -415,11 +492,13 @@ impl Monitor { ); } - if let Some(alertmanager_targets) = &self - .prometheus - .as_ref() - .and_then(|p| p.alertmanager_targets.as_ref()) - { + if let Some(alertmanager_targets) = &self.prometheus.as_ref().and_then(|p| { + p.alertmanager_targets.as_ref().cloned().or_else(|| { + self.alertmanager + .as_ref() + .map(|am| vec![format!("{}:{}", am.host, am.port)]) + }) + }) { let am_static_targets: Vec = alertmanager_targets .iter() .map(|t| Value::String(t.clone())) @@ -454,6 +533,113 @@ impl Monitor { Ok(prometheus_config_file) } + pub fn gen_alertmanager_config(&self, cluster_name: &str) -> anyhow::Result { + let mut alertmanager_config_map = load_yaml_config_template(ALERTMANAGER_CONFIG_TEMPLATE)?; + let receiver_name = if self + .alertmanager + .as_ref() + .and_then(|alertmanager| alertmanager.feishu_robot_urls.as_ref()) + .is_some_and(|urls| !urls.is_empty()) + { + "alertmanager_webhook_adapter" + } else { + "null" + }; + + if let Some(route) = alertmanager_config_map + .get_mut("route") + .and_then(|v| v.as_mapping_mut()) + { + route.insert( + Value::String("receiver".to_string()), + Value::String(receiver_name.to_string()), + ); + } + + let mut receivers = vec![{ + let mut receiver = serde_yaml::Mapping::new(); + receiver.insert( + Value::String("name".to_string()), + Value::String("null".to_string()), + ); + Value::Mapping(receiver) + }]; + + if let Some(alertmanager) = &self.alertmanager { + let webhook_entries = alertmanager + .feishu_robot_urls + .clone() + .unwrap_or_default() + .into_iter() + .filter_map(|url| Self::extract_feishu_robot_token(&url)) + .map(|token| { + let mut webhook_entry = serde_yaml::Mapping::new(); + webhook_entry.insert( + Value::String("url".to_string()), + Value::String(format!( + "http://{}:{}/webhook/send?channel_type=feishu&token={token}", + alertmanager.host, alertmanager.webhook_adapter_port + )), + ); + Value::Mapping(webhook_entry) + }) + .collect_vec(); + if !webhook_entries.is_empty() { + let mut receiver = serde_yaml::Mapping::new(); + receiver.insert( + Value::String("name".to_string()), + Value::String("alertmanager_webhook_adapter".to_string()), + ); + receiver.insert( + Value::String("webhook_configs".to_string()), + Value::Sequence(webhook_entries), + ); + receivers.push(Value::Mapping(receiver)); + } + } + + alertmanager_config_map.insert("receivers".to_string(), Value::Sequence(receivers)); + + let alertmanager_config_file = upload_dir() + .join(cluster_name) + .join(MONITOR_DIR) + .join(ALERTMANAGER_CONFIG_FILE); + if let Some(parent) = alertmanager_config_file.parent() { + fs::create_dir_all(parent)?; + } + let file = File::create(&alertmanager_config_file)?; + serde_yaml::to_writer(file, &alertmanager_config_map)?; + Ok(alertmanager_config_file) + } + + pub fn gen_alertmanager_webhook_adapter_template( + &self, + cluster_name: &str, + ) -> anyhow::Result { + if self.alertmanager.is_none() { + anyhow::bail!("alertmanager config not found"); + } + let template_path = config_template(ALERTMANAGER_WEBHOOK_ADAPTER_FEISHU_TEMPLATE)?; + let target_path = upload_dir() + .join(cluster_name) + .join(MONITOR_DIR) + .join(ALERTMANAGER_WEBHOOK_ADAPTER_FEISHU_TEMPLATE); + if let Some(parent) = target_path.parent() { + fs::create_dir_all(parent)?; + } + fs::copy(template_path, &target_path)?; + Ok(target_path) + } + + fn extract_feishu_robot_token(url: &str) -> Option { + let trimmed = url.trim().trim_end_matches('/'); + trimmed + .rsplit("/hook/") + .next() + .filter(|token| !token.is_empty() && *token != trimmed) + .map(|token| token.to_string()) + } + pub fn gen_grafana_datasource_config(&self, cluster_name: &str) -> anyhow::Result { let prometheus = &self.prometheus; let prometheus_url = format!( @@ -485,3 +671,253 @@ impl Monitor { Ok(grafana_ds_config) } } + +#[cfg(test)] +mod tests { + use super::{Alertmanager, Grafana, Monitor, Prometheus}; + use crate::config::{CONFIG_PATH_DIR, UPLOAD_PATH_DIR}; + use serde_yaml::Value; + use std::collections::HashMap; + use std::fs; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn test_upload_dir(name: &str) -> PathBuf { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("cluster-mgr-{name}-{suffix}")); + fs::create_dir_all(&dir).unwrap(); + dir + } + + fn set_template_dirs(upload_dir: &std::path::Path) { + std::env::set_var( + CONFIG_PATH_DIR, + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("config") + .to_string_lossy() + .to_string(), + ); + std::env::set_var(UPLOAD_PATH_DIR, upload_dir.to_string_lossy().to_string()); + } + + #[test] + fn grafana_version_detection_supports_legacy_and_modern_names() { + assert_eq!( + Monitor::grafana_major_version( + "https://dl.grafana.com/oss/release/grafana-9.3.6.linux-amd64.tar.gz" + ), + Some(9) + ); + assert_eq!( + Monitor::grafana_major_version( + "https://dl.grafana.com/grafana/release/13.0.1+security-01/grafana_13.0.1+security-01_25720641773_linux_amd64.tar.gz" + ), + Some(13) + ); + } + + #[test] + fn grafana_legacy_config_keeps_full_template() { + let upload_dir = test_upload_dir("grafana-legacy"); + set_template_dirs(&upload_dir); + let monitor = Monitor { + data_dir: None, + prometheus: None, + alertmanager: None, + grafana: Some(Grafana { + download_url: "https://dl.grafana.com/oss/release/grafana-9.3.6.linux-amd64.tar.gz" + .to_string(), + port: 3301, + host: "127.0.0.1".to_string(), + }), + node_exporter: None, + eloq_metrics: None, + }; + + let path = monitor.gen_grafana_config("legacy").unwrap(); + let content = fs::read_to_string(path).unwrap(); + assert!(content.contains("[security]")); + assert!(content.contains("http_port=3301")); + } + + #[test] + fn grafana_modern_config_only_overrides_required_fields() { + let upload_dir = test_upload_dir("grafana-modern"); + set_template_dirs(&upload_dir); + let monitor = Monitor { + data_dir: None, + prometheus: None, + alertmanager: None, + grafana: Some(Grafana { + download_url: + "https://dl.grafana.com/grafana/release/13.0.1+security-01/grafana_13.0.1+security-01_25720641773_linux_amd64.tar.gz" + .to_string(), + port: 3301, + host: "127.0.0.1".to_string(), + }), + node_exporter: None, + eloq_metrics: None, + }; + + let path = monitor.gen_grafana_config("modern").unwrap(); + let content = fs::read_to_string(path).unwrap(); + assert!(content.contains("[server]")); + assert!(content.contains("http_port=3301")); + assert!(!content.contains("[security]")); + } + + #[test] + fn prometheus_config_auto_wires_alertmanager_target() { + let upload_dir = test_upload_dir("prometheus-alertmanager"); + set_template_dirs(&upload_dir); + let monitor = Monitor { + data_dir: None, + prometheus: Some(Prometheus { + download_url: "https://example.com/prometheus.tar.gz".to_string(), + port: 9500, + host: "127.0.0.1".to_string(), + retention_time: None, + retention_size: None, + remote_write_urls: None, + alertmanager_targets: None, + alert_thresholds: None, + }), + alertmanager: Some(Alertmanager { + download_url: "https://example.com/alertmanager.tar.gz".to_string(), + port: 9093, + host: "127.0.0.1".to_string(), + feishu_robot_urls: None, + webhook_adapter_download_url: super::ALERTMANAGER_WEBHOOK_ADAPTER_DEFAULT_URL + .to_string(), + webhook_adapter_port: super::ALERTMANAGER_WEBHOOK_ADAPTER_PORT, + }), + grafana: None, + node_exporter: Some(super::Exporter { + url: "https://example.com/node_exporter.tar.gz".to_string(), + port: 9200, + }), + eloq_metrics: None, + }; + + let path = monitor + .gen_prometheus_config("prom", HashMap::>::new()) + .unwrap(); + let content = fs::read_to_string(path).unwrap(); + assert!(content.contains("127.0.0.1:9093")); + } + + #[test] + fn alertmanager_extracts_feishu_robot_token() { + let upload_dir = test_upload_dir("alertmanager-feishu-token"); + set_template_dirs(&upload_dir); + assert_eq!( + Monitor::extract_feishu_robot_token("https://open.feishu.cn/open-apis/bot/v2/hook/abc"), + Some("abc".to_string()) + ); + assert_eq!(Monitor::extract_feishu_robot_token("invalid"), None); + } + + #[test] + fn alertmanager_config_targets_webhook_adapter() { + let upload_dir = test_upload_dir("alertmanager-config"); + set_template_dirs(&upload_dir); + let monitor = Monitor { + data_dir: None, + prometheus: None, + alertmanager: Some(Alertmanager { + download_url: "https://example.com/alertmanager.tar.gz".to_string(), + port: 9093, + host: "127.0.0.1".to_string(), + feishu_robot_urls: Some(vec![ + "https://open.feishu.cn/open-apis/bot/v2/hook/abc".to_string() + ]), + webhook_adapter_download_url: super::ALERTMANAGER_WEBHOOK_ADAPTER_DEFAULT_URL + .to_string(), + webhook_adapter_port: 18080, + }), + grafana: None, + node_exporter: None, + eloq_metrics: None, + }; + + let path = monitor.gen_alertmanager_config("alertmanager").unwrap(); + let content = fs::read_to_string(path).unwrap(); + let yaml: HashMap = serde_yaml::from_str(&content).unwrap(); + assert!(content.contains("/webhook/send?channel_type=feishu&token=abc")); + assert_eq!( + yaml.get("route") + .and_then(Value::as_mapping) + .and_then(|m| m.get(Value::String("receiver".to_string()))) + .and_then(Value::as_str), + Some("alertmanager_webhook_adapter") + ); + } + + #[test] + fn alertmanager_webhook_adapter_template_is_staged() { + let upload_dir = test_upload_dir("alertmanager-template"); + set_template_dirs(&upload_dir); + let monitor = Monitor { + data_dir: None, + prometheus: None, + alertmanager: Some(Alertmanager { + download_url: "https://example.com/alertmanager.tar.gz".to_string(), + port: 9093, + host: "127.0.0.1".to_string(), + feishu_robot_urls: Some(vec![ + "https://open.feishu.cn/open-apis/bot/v2/hook/abc".to_string() + ]), + webhook_adapter_download_url: super::ALERTMANAGER_WEBHOOK_ADAPTER_DEFAULT_URL + .to_string(), + webhook_adapter_port: 18080, + }), + grafana: None, + node_exporter: None, + eloq_metrics: None, + }; + + let path = monitor + .gen_alertmanager_webhook_adapter_template("alertmanager") + .unwrap(); + let content = fs::read_to_string(path).unwrap(); + assert!(content.contains("EloqKV")); + assert!(content.contains("**实例详情**")); + assert!(content.contains("severity.tag")); + } + + #[test] + fn alert_rules_use_cluster_ops_and_embed_cluster_name() { + let upload_dir = test_upload_dir("alert-rules-cluster-ops"); + set_template_dirs(&upload_dir); + let monitor = Monitor { + data_dir: None, + prometheus: Some(Prometheus { + download_url: "https://example.com/prometheus.tar.gz".to_string(), + port: 9500, + host: "127.0.0.1".to_string(), + retention_time: None, + retention_size: None, + remote_write_urls: None, + alertmanager_targets: None, + alert_thresholds: None, + }), + alertmanager: None, + grafana: None, + node_exporter: None, + eloq_metrics: None, + }; + + let path = monitor.gen_alert_rules("test-e2e").unwrap(); + let content = fs::read_to_string(path).unwrap(); + + assert!(content.contains("alert: eloqkv_cluster_ops_too_low")); + assert!(content.contains("sum(rate(redis_command_aggregated_total[1m])) < 100000")); + assert!(content.contains("EloqKV cluster test-e2e OPS has been below 100000 for 1 minute")); + assert!(!content.contains("eloqkv_instance_ops_too_low")); + assert!(!content.contains("sum by (instance) (rate(redis_command_aggregated_total[1m]))")); + assert!(!content.contains("${CLUSTER_NAME}")); + } +} diff --git a/src/cluster_mgr/src/state/state_mgr.rs b/src/cluster_mgr/src/state/state_mgr.rs index 8f89ff49..11aa357c 100644 --- a/src/cluster_mgr/src/state/state_mgr.rs +++ b/src/cluster_mgr/src/state/state_mgr.rs @@ -586,24 +586,68 @@ impl StateMgr { #[cfg(test)] mod tests { + use crate::cli::HOME_DIR; use crate::state::cluster_index_operation::ClusterIndexOperation; use crate::state::state_base::{QueryCondition, StateOperation}; use crate::state::state_mgr::{StateMgr, CLUSTER_INDEX_STATE, ELOQ_CLUSTER_MGR_SCHEMA_PATH}; use sqlx::testing::TestTermination; + use std::fs; use std::path::PathBuf; - use std::sync::LazyLock; + use std::sync::{LazyLock, Mutex, MutexGuard}; use tracing::Level; + use uuid::Uuid; static SETUP: LazyLock<()> = LazyLock::new(|| { tracing_subscriber::fmt() .with_max_level(Level::DEBUG) .init(); }); + static TEST_MUTEX: LazyLock> = LazyLock::new(|| Mutex::new(())); fn setup() { assert!(SETUP.is_success()); } + fn test_guard() -> MutexGuard<'static, ()> { + TEST_MUTEX.lock().expect("test mutex poisoned") + } + + struct EnvVarGuard { + key: &'static str, + previous: Option, + } + + impl EnvVarGuard { + fn set_path(key: &'static str, value: &std::path::Path) -> Self { + let previous = std::env::var(key).ok(); + std::env::set_var(key, value); + Self { key, previous } + } + + fn set_string(key: &'static str, value: &str) -> Self { + let previous = std::env::var(key).ok(); + std::env::set_var(key, value); + Self { key, previous } + } + } + + impl Drop for EnvVarGuard { + fn drop(&mut self) { + if let Some(prev) = &self.previous { + std::env::set_var(self.key, prev); + } else { + std::env::remove_var(self.key); + } + } + } + + fn setup_test_home() -> (PathBuf, EnvVarGuard) { + let home = std::env::temp_dir().join(format!("eloqctl-test-{}", Uuid::new_v4())); + fs::create_dir_all(&home).expect("create test home"); + let guard = EnvVarGuard::set_path(HOME_DIR, &home); + (home, guard) + } + pub fn schema_path() -> String { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let config_path = manifest_dir.join("config"); @@ -611,12 +655,15 @@ mod tests { schema_path } + #[allow(clippy::await_holding_lock)] #[tokio::test(flavor = "multi_thread", worker_threads = 1)] pub async fn test_db_init() { + let _guard = test_guard(); setup(); + let (home, _home_guard) = setup_test_home(); let schema_path = schema_path(); println!("schema_path {schema_path:?}"); - std::env::set_var(ELOQ_CLUSTER_MGR_SCHEMA_PATH, schema_path.clone()); + let _schema_guard = EnvVarGuard::set_string(ELOQ_CLUSTER_MGR_SCHEMA_PATH, &schema_path); let state_mgr = StateMgr::new(schema_path).await; assert!(state_mgr.is_ok()); let all_tables = state_mgr.unwrap().list_tables().await; @@ -633,14 +680,18 @@ mod tests { "t_topology_log", ] ); + fs::remove_dir_all(home).ok(); } + #[allow(clippy::await_holding_lock)] #[tokio::test(flavor = "multi_thread", worker_threads = 1)] pub async fn test_state_load() { + let _guard = test_guard(); setup(); + let (home, _home_guard) = setup_test_home(); let schema_path = schema_path(); println!("schema_path {schema_path:?}"); - std::env::set_var(ELOQ_CLUSTER_MGR_SCHEMA_PATH, schema_path.clone()); + let _schema_guard = EnvVarGuard::set_string(ELOQ_CLUSTER_MGR_SCHEMA_PATH, &schema_path); let state_mgr_rs = StateMgr::new(schema_path).await; assert!(state_mgr_rs.is_ok()); let state_mgr = state_mgr_rs.unwrap(); @@ -652,5 +703,6 @@ mod tests { assert!(deployment_result.is_ok()); let deployment_vec = deployment_result.unwrap(); assert!(deployment_vec.is_empty()); + fs::remove_dir_all(home).ok(); } } diff --git a/src/rest_api/src/handler.rs b/src/rest_api/src/handler.rs index 21bf3a6c..fdf36cb4 100644 --- a/src/rest_api/src/handler.rs +++ b/src/rest_api/src/handler.rs @@ -4,7 +4,7 @@ use actix_web::{get, post, web, HttpResponse, Responder}; use anyhow::anyhow; use cluster_mgr::cli::task::group::Config; use cluster_mgr::cli::task::task_base::TaskId; -use cluster_mgr::cli::SubCommand; +use cluster_mgr::cli::{MonitorCommand, SubCommand}; use cluster_mgr::config::config_base::{DeployConfig, DEPLOYMENT_CHECK_SUCCESS_TASK}; use serde_json::{json, Value}; use std::collections::HashMap; @@ -86,12 +86,18 @@ fn build_command_from_str(cmd_str: &str, cluster: Option) -> SubCommand topology_file: "_NONE".to_string(), }, "start_monitor" => SubCommand::Monitor { - cluster: cluster.unwrap(), - command: "start".to_string(), + cluster: Some(cluster.clone().unwrap()), + command: MonitorCommand::Start { + cluster: Some(cluster.unwrap()), + components: Vec::new(), + }, }, "stop_monitor" => SubCommand::Monitor { - cluster: cluster.unwrap(), - command: "stop".to_string(), + cluster: Some(cluster.clone().unwrap()), + command: MonitorCommand::Stop { + cluster: Some(cluster.unwrap()), + components: Vec::new(), + }, }, "start_log" => SubCommand::LogService { cluster: cluster.unwrap(), diff --git a/tests/README.md b/tests/README.md index 220ea396..bf5b7633 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,233 +1,238 @@ # E2E Tests -Deploys an EloqKV cluster in Docker containers via `eloqctl launch`, then runs -multi-SDK stress workloads against it. Every test uses both standalone and cluster -Redis clients with TLS enabled. +这套测试只讲两种用法。 -## Quick Start +## 方式 1：用 CLI 跑完整测试 + +先构建本地 `eloqctl`： ```sh cd /home/starrysky/workspace/eloqdata-kernel/eloq_waiter +cargo build -p cluster_mgr +``` -# Build eloqctl (one time) -scripts/install-dev.sh +统一入口： -# Run everything: launch → Python → Go → TS → remove -bash tests/e2e/cmd_stress_test.sh +```sh +tests/e2e/devctl.sh --help ``` -The default stress duration is now `300` seconds per SDK step. The appended -high-concurrency `INFO` burst in each SDK step also defaults to `300` seconds. - -## Run specific steps +最常用命令： ```sh -# Launch only -STEPS=launch bash tests/e2e/cmd_stress_test.sh +# 启动 Docker 环境 +tests/e2e/devctl.sh env-up + +# 构建 + 启动环境 + 安装到 control node + 生成 topology + launch +tests/e2e/devctl.sh full -# Stress only (against already-running cluster) -STEPS=py-stress,go-stress,ts-stress bash tests/e2e/cmd_stress_test.sh +# 查看集群和 monitor 状态 +tests/e2e/devctl.sh status -# Override the default 300-second stress duration -DURATION_SECONDS=120 INFO_ONLY_DURATION_SECONDS=120 \ - STEPS=py-stress,go-stress,ts-stress bash tests/e2e/cmd_stress_test.sh +# 只升级 Grafana +tests/e2e/devctl.sh grafana-update + +# 运行完整 stress test +tests/e2e/devctl.sh stress + +# 只跑 SDK stress，流量都在容器里，不在宿主机 +tests/e2e/devctl.sh stress py-stress,go-stress,ts-stress + +# 删除 Docker 环境 +tests/e2e/devctl.sh env-down ``` -## Control Node Usage +说明： + +- `devctl.sh stress` 实际调用 `tests/e2e/cmd_stress_test.sh` +- Python 压测跑在 `stress-python` +- Go 压测跑在 `stress-go` +- TypeScript 压测跑在 `stress-ts` +- `env-up` 默认复用本地已有镜像；如果你要强制重建，用 `FORCE_DOCKER_BUILD=1 tests/e2e/devctl.sh env-up` +- `env-up` 只负责启动一个新的 Docker 测试环境；control node 里的 `eloqctl` 运行目录会是全新的 + +## 方式 2：用 CLI 启动环境，再登录 control node 手动跑 eloqctl -The Docker E2E environment uses `eloq-node-4` as the control node. Install the -current host-built `eloqctl` into that container with: +### 1. 启动环境 ```sh -tests/install_control_eloqctl.sh +cd /home/starrysky/workspace/eloqdata-kernel/eloq_waiter +cargo build -p cluster_mgr +tests/e2e/devctl.sh env-up +tests/e2e/devctl.sh install-control +tests/e2e/devctl.sh render-topology +``` + +### 2. 登录 control node + +```sh +tests/e2e/devctl.sh control-shell ``` -Then log in directly: +等价命令： ```sh -ssh -i "tests/docker_ha/id_ed25519" -p 2224 eloq@127.0.0.1 +ssh -i tests/docker_ha/id_ed25519 -p 2224 eloq@127.0.0.1 ``` -Inside `node4`, `eloqctl` is installed at `/usr/local/bin/eloqctl` and uses -`/home/eloq/.eloqctl` as `ELOQCTL_HOME`. +control node 里的关键路径： -### Generate launch topology inside `node4` +- 仓库：`/workspace/eloq_waiter` +- `eloqctl`：`/usr/local/bin/eloqctl` +- `ELOQCTL_HOME`：`/home/eloq/.eloqctl` +- 渲染后的 topology：`/home/eloq/topology.generated.yaml` -The template topology lives at `tests/e2e/topology.yaml`. From a shell inside -`node4`, generate the control-node-ready topology and launch from it: +### 3. 手动 launch / update / status + +在 control node 里执行： ```sh -cd /workspace/eloq_waiter -export DOCKER_E2E_DIR=/workspace/eloq_waiter/tests/docker_ha -export REPO_ROOT=/workspace/eloq_waiter -source tests/docker_env.sh -render_topology_for_control tests/e2e/topology.yaml tests/e2e/topology.generated.yaml -eloqctl launch --skip-deps tests/e2e/topology.generated.yaml +eloqctl stop test-e2e --all --force || true +eloqctl remove test-e2e --force || true +eloqctl launch --skip-deps /home/eloq/topology.generated.yaml ``` -To use a different EloqKV version, set `ELOQKV_VERSION` before sourcing: +查看状态： ```sh -export ELOQKV_VERSION=1.3.0 +eloqctl status test-e2e --wait 180 +eloqctl monitor status --cluster test-e2e ``` -### Check status and connect - -After launch, `eloqctl status ` prints both the status table and a -ready-to-copy `redis-cli` command: +手动升级 Grafana： ```sh -eloqctl status test-e2e +eloqctl monitor update --cluster test-e2e \ + --component grafana \ + --url 'https://dl.grafana.com/grafana/release/13.0.1+security-01/grafana_13.0.1+security-01_25720641773_linux_amd64.tar.gz' ``` -## Configuration +在现有集群上安装 Alertmanager： -| Variable | Default | Description | -|----------|---------|-------------| -| `STEPS` | `launch,py-stress,go-stress,ts-stress,remove` | Comma-separated steps | -| `WORKERS` | `16` | Total workers (split evenly: standalone / cluster client) | -| `DURATION_SECONDS` | `300` | Main stress duration per SDK step | -| `INFO_ONLY_DURATION_SECONDS` | `300` | Duration of the appended high-concurrency `INFO` burst | -| `INFO_ONLY_WORKERS` | `64` | Worker count for the appended `INFO` burst | -| `INFO_ONLY_INFLIGHT` | `16` | Inflight slots per worker for the appended `INFO` burst | -| `INFO_ONLY_REPEAT` | `50` | Repeat count per round for the appended `INFO` burst | -| `KEY_COUNT` | `256` | Preloaded key count | -| `CMD_TIMEOUT` | `5` | Per-command timeout (seconds) | -| `PROGRESS_INTERVAL` | `5` | Progress report interval (seconds) | -| `TLS_ENABLED` | `1` | Enable TLS on cluster and clients | -| `SKIP_DEPS` | `1` | Skip OS dep installation on nodes | +```sh +eloqctl monitor update --cluster test-e2e \ + --component alertmanager \ + --url 'https://github.com/prometheus/alertmanager/releases/download/v0.32.1/alertmanager-0.32.1.linux-amd64.tar.gz' +``` -## Directory Layout +再次更新 Alertmanager： -``` -tests/ -├── README.md -├── docker_env.sh # shared helpers: Docker Compose, SSH, MinIO -├── docker_ha/ -│ ├── docker-compose.yaml # 4-node Ubuntu + MinIO + stress containers -│ ├── Dockerfile # SSH image for eloq nodes -│ ├── Dockerfile.stress # Python 3.13 + redis-py -│ ├── Dockerfile.stress_go # Go 1.24 + go-redis/v9 -│ ├── Dockerfile.stress_ts # Node 22 + ioredis -│ ├── id_ed25519 / id_ed25519.pub # auto-generated SSH key -│ └── authorized_keys -└── e2e/ - ├── cmd_stress_test.sh # main entry point ★ - ├── cmd_stress_py/ - │ ├── main.py # Python full-command stress client - │ └── requirements.txt # pinned redis-py dependency (5.0.1) - ├── cmd_stress_go/ - │ ├── main.go # Go full-command stress client - │ ├── go.mod / go.sum - ├── cmd_stress_ts/ - │ ├── main.ts # TypeScript full-command stress client - │ ├── package.json / package-lock.json / tsconfig.json - └── topology.yaml # cluster topology template +```sh +eloqctl monitor update --cluster test-e2e \ + --component alertmanager \ + --url 'https://github.com/prometheus/alertmanager/releases/download/v0.32.1/alertmanager-0.32.1.linux-amd64.tar.gz' ``` -## Command Coverage +在现有集群上安装 Alertmanager，并同时安装 `alertmanager-webhook-adapter`： -Each SDK stress test covers **104 Redis commands** across all families -(string, hash, list, set, sorted-set, generic/key, server/connection). +```sh +eloqctl monitor update --cluster test-e2e \ + --component alertmanager \ + --url 'https://github.com/prometheus/alertmanager/releases/download/v0.32.1/alertmanager-0.32.1.linux-amd64.tar.gz' +``` -Every test runs **half the workers with a standalone client** (direct to master) -and **half with a cluster-aware client** (auto slot routing). Results for both -modes are reported separately. The standalone client always targets the master. -Only the cluster-aware client is configured to read from replicas. +在现有集群上安装 Alertmanager，并同时启用飞书告警转发： -After the normal mixed-command phase, each SDK appends a dedicated, -high-concurrency `INFO` burst. +```sh +eloqctl monitor update --cluster test-e2e \ + --component alertmanager \ + --url 'https://github.com/prometheus/alertmanager/releases/download/v0.32.1/alertmanager-0.32.1.linux-amd64.tar.gz' \ + --feishu-robot-url 'https://open.feishu.cn/open-apis/bot/v2/hook/xxx' +``` -TLS is enabled by default with self-signed certs (`rejectUnauthorized: false` / -`ssl_cert_reqs=CERT_NONE` / `InsecureSkipVerify`). +这条命令还会一并部署 `alertmanager-webhook-adapter`，并下发仓库内置的飞书中文模板： -## Troubleshooting +- 模板语言：`zh` +- 默认签名：`EloqKV` +- 模板文件：`src/cluster_mgr/config/feishu.zh.tmpl` +- 远端部署目录：`/home/eloq/test-e2e/alertmanager-webhook-adapter/templates/feishu.zh.tmpl` -If launch fails, rebuild Docker images: +再次更新 Alertmanager，或用相同命令恢复一次失败安装： ```sh -cd tests/docker_ha && docker compose build --no-cache +eloqctl monitor update --cluster test-e2e \ + --component alertmanager \ + --url 'https://github.com/prometheus/alertmanager/releases/download/v0.32.1/alertmanager-0.32.1.linux-amd64.tar.gz' \ + --feishu-robot-url 'https://open.feishu.cn/open-apis/bot/v2/hook/xxx' ``` -Check cluster health: +安装完后再次检查： ```sh -eloqctl status test-e2e --wait 30 +eloqctl monitor status --cluster test-e2e ``` -Logs auto-clean unless `KEEP_LOGS=1` is set. - -## Connecting from Host +如果你以前装过旧的独立 `PrometheusAlert`，想把残留进程和目录删干净，可以在 control node 里手工执行： -The cluster nodes are exposed on localhost ports. After `docker compose up` and -a successful `eloqctl launch`: +```sh +ssh -i /home/eloq/.ssh/id_ed25519 eloq@172.28.10.14 \ + "pkill -f '/home/eloq/test-e2e/prometheusalert/PrometheusAlert' || true; \ + rm -rf /home/eloq/test-e2e/prometheusalert" +``` -| Node | Host Port | Docker IP | -|------|-----------|-----------| -| node-1 | `127.0.0.1:16371` | `172.28.10.11:6379` | -| node-2 | `127.0.0.1:16372` | `172.28.10.12:6379` | -| node-3 | `127.0.0.1:16373` | `172.28.10.13:6379` | +这一步只用于清理旧实现残留。新的飞书告警链路部署目录是 `/home/eloq/test-e2e/alertmanager-webhook-adapter`。 -### Check cluster topology +导出 topology： ```sh -redis-cli -h 127.0.0.1 -p 16371 --tls --insecure -a testpass CLUSTER NODES +eloqctl export test-e2e --output /home/eloq/test-e2e-export.yaml ``` -The output shows which node is `master` and which is `slave`: +### 4. 看 monitor 网页 -``` - 172.28.10.11:6379@16380 myself,slave ... - 172.28.10.12:6379@16380 master ... -``` +宿主机浏览器访问： -### Connect with redis-py (Python) +- Grafana: `http://127.0.0.1:13301` +- Prometheus: `http://127.0.0.1:19500` +- Alertmanager: `http://127.0.0.1:19093`，安装 `alertmanager` 后可访问 +- Alertmanager Webhook Adapter: `http://127.0.0.1:18080`，安装 `alertmanager` 后可访问 -```python -import ssl, redis -TLS = {'ssl': True, 'ssl_cert_reqs': ssl.CERT_NONE, 'ssl_check_hostname': False} +Grafana 默认账号密码： -# standalone — connect directly to one node -r = redis.Redis(host='127.0.0.1', port=16371, password='testpass', **TLS) +```text +admin / admin +``` -# cluster mode — auto-routes to correct node -from redis.cluster import RedisCluster, ClusterNode -rc = RedisCluster(startup_nodes=[ - ClusterNode('127.0.0.1', 16371), - ClusterNode('127.0.0.1', 16372), -], password='testpass', **TLS) +也可以用命令验证： + +```sh +curl -fsS http://127.0.0.1:13301/login >/dev/null +curl -fsS http://127.0.0.1:19500/-/healthy +curl -fsS http://127.0.0.1:19093/-/healthy +curl -fsS http://127.0.0.1:18080 >/dev/null ``` -### Connect with go-redis (Go) +### 5. 删除环境 -```go -// standalone -c := redis.NewClient(&redis.Options{ - Addr: "127.0.0.1:16371", Password: "testpass", - TLSConfig: &tls.Config{InsecureSkipVerify: true}, -}) +宿主机执行： -// cluster -cc := redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{"127.0.0.1:16371", "127.0.0.1:16372"}, - Password: "testpass", - TLSConfig: &tls.Config{InsecureSkipVerify: true}, -}) +```sh +tests/e2e/devctl.sh env-down ``` -### Connect with ioredis (TypeScript) +## Stress Test 常用变量 + +`tests/e2e/cmd_stress_test.sh` 支持这些常用覆盖项： -```typescript -// standalone -const r = new Redis({ - host: "127.0.0.1", port: 16371, password: "testpass", - tls: { rejectUnauthorized: false }, -}); +| Variable | Default | +|----------|---------| +| `STEPS` | `launch,monitor-update,eloqctl-mutate,py-stress,go-stress,ts-stress,remove` | +| `DURATION_SECONDS` | `300` | +| `INFO_ONLY_DURATION_SECONDS` | `300` | +| `WORKERS` | `16` | +| `INFLIGHT` | `4` | +| `KEY_COUNT` | `256` | +| `CMD_TIMEOUT` | `5` | +| `TLS_ENABLED` | `1` | +| `SKIP_DEPS` | `1` | -// cluster -const c = new Cluster([ - { host: "127.0.0.1", port: 16371 }, - { host: "127.0.0.1", port: 16372 }, -], { redisOptions: { password: "testpass", - tls: { rejectUnauthorized: false } } }); +例子： + +```sh +STEPS=py-stress,go-stress,ts-stress \ + DURATION_SECONDS=15 \ + INFO_ONLY_DURATION_SECONDS=15 \ + WORKERS=4 \ + INFLIGHT=2 \ + tests/e2e/devctl.sh stress ``` diff --git a/tests/docker_env.sh b/tests/docker_env.sh index 54b98c11..e634fc72 100755 --- a/tests/docker_env.sh +++ b/tests/docker_env.sh @@ -1,21 +1,33 @@ #!/bin/bash -set -eo pipefail +_docker_env_script_dir() { + if [ -n "${BASH_SOURCE[0]:-}" ]; then + cd "$(dirname "${BASH_SOURCE[0]}")" && pwd + return + fi + if [ -n "${ZSH_VERSION:-}" ]; then + cd "$(dirname "${(%):-%x}")" && pwd + return + fi + pwd +} -DOCKER_E2E_DIR="${DOCKER_E2E_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/docker_ha" && pwd)}" +DOCKER_ENV_SCRIPT_DIR="${DOCKER_ENV_SCRIPT_DIR:-$(_docker_env_script_dir)}" +DOCKER_E2E_DIR="${DOCKER_E2E_DIR:-$(cd "${DOCKER_ENV_SCRIPT_DIR}/docker_ha" && pwd)}" REPO_ROOT="${REPO_ROOT:-$(cd "${DOCKER_E2E_DIR}/../.." && pwd)}" export ELOQCTL_HOME="${ELOQCTL_HOME:-${HOME}/.eloqctl}" ELOQCTL="${ELOQCTL:-${ELOQCTL_HOME}/bin/cluster_mgr}" ELOQCTL_DOCKER_SSH_KEY="${ELOQCTL_DOCKER_SSH_KEY:-${DOCKER_E2E_DIR}/id_ed25519}" ELOQCTL_CONTROL_SSH_KEY="${ELOQCTL_CONTROL_SSH_KEY:-/home/eloq/.ssh/id_ed25519}" -CONTROL_NODE_SERVICE="${CONTROL_NODE_SERVICE:-eloq-node-4}" +CONTROL_NODE_SERVICE="${CONTROL_NODE_SERVICE:-eloq-control}" CONTROL_REPO_ROOT="${CONTROL_REPO_ROOT:-/workspace/eloq_waiter}" CONTROL_ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME:-/home/eloq/.eloqctl}" CONTROL_ELOQCTL="${CONTROL_ELOQCTL:-${CONTROL_ELOQCTL_HOME}/bin/cluster_mgr}" -HOST_ELOQCTL_HOME="${HOST_ELOQCTL_HOME:-${REPO_ROOT}/.tmp-eloqctl}" export ELOQCTL_DOCKER_SSH_KEY CLEANUP_TIMEOUT_SECONDS="${CLEANUP_TIMEOUT_SECONDS:-20}" +FORCE_DOCKER_BUILD="${FORCE_DOCKER_BUILD:-0}" ELOQKV_VERSION="${ELOQKV_VERSION:-1.2.2}" +ELOQCTL_FEISHU_ROBOT_URL="${ELOQCTL_FEISHU_ROBOT_URL:-}" MINIO_ALIAS="${MINIO_ALIAS:-e2e-minio}" MINIO_ENDPOINT="${MINIO_ENDPOINT:-http://127.0.0.1:19000}" MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}" @@ -38,11 +50,40 @@ compose() { fi } +docker_image_exists() { + docker image inspect "$1" >/dev/null 2>&1 +} + +ensure_docker_images() { + if [ "${FORCE_DOCKER_BUILD}" = "1" ]; then + echo "[docker] Build Ubuntu SSH and stress containers (forced)" + COMPOSE_PROGRESS=plain compose build + return + fi + + local required_images=( + "eloqctl-e2e-ubuntu:24.04" + "eloqctl-e2e-stress:latest" + "eloqctl-e2e-stress-go:latest" + "eloqctl-e2e-stress-ts:latest" + ) + local image + for image in "${required_images[@]}"; do + if ! docker_image_exists "${image}"; then + echo "[docker] Build Ubuntu SSH and stress containers" + COMPOSE_PROGRESS=plain compose build + return + fi + done + + echo "[docker] Reuse existing Ubuntu SSH and stress images" +} + compose_down() { if docker compose version >/dev/null 2>&1; then - timeout --kill-after=5s "${CLEANUP_TIMEOUT_SECONDS}s" docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" down -v >/dev/null 2>&1 || true + timeout --kill-after=5s "${CLEANUP_TIMEOUT_SECONDS}s" docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" down -v --remove-orphans >/dev/null 2>&1 || true else - timeout --kill-after=5s "${CLEANUP_TIMEOUT_SECONDS}s" docker-compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" down -v >/dev/null 2>&1 || true + timeout --kill-after=5s "${CLEANUP_TIMEOUT_SECONDS}s" docker-compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" down -v --remove-orphans >/dev/null 2>&1 || true fi } @@ -58,10 +99,11 @@ ssh_cmd() { "${@:2}" } -ensure_dev_eloqctl() { - if [ ! -d "${ELOQCTL_HOME}/config" ] || [ ! -x "${ELOQCTL}" ]; then - "${REPO_ROOT}/scripts/install-dev.sh" - fi +control_ssh_cmd() { + local remote_cmd + printf -v remote_cmd '%q ' \ + env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" "${CONTROL_ELOQCTL}" "$@" + ssh_cmd 2224 "bash -lc $(printf '%q' "${remote_cmd}")" } ensure_ssh_key() { @@ -71,6 +113,20 @@ ensure_ssh_key() { cp "${ELOQCTL_DOCKER_SSH_KEY}.pub" "${DOCKER_E2E_DIR}/authorized_keys" } +reset_control_runtime_state_in_container() { + compose exec -T "${CONTROL_NODE_SERVICE}" bash -lc " + set -eu + rm -rf '${CONTROL_ELOQCTL_HOME}' + install -d -m 755 -o eloq -g eloq \ + '${CONTROL_ELOQCTL_HOME}' \ + '${CONTROL_ELOQCTL_HOME}/bin' \ + '${CONTROL_ELOQCTL_HOME}/db' \ + '${CONTROL_ELOQCTL_HOME}/download' \ + '${CONTROL_ELOQCTL_HOME}/logs' \ + '${CONTROL_ELOQCTL_HOME}/upload' + " +} + render_topology() { local source_topology="$1" local rendered_topology="$2" @@ -84,6 +140,7 @@ render_topology_for_control() { local rendered_topology="$2" sed -e "s|\${ELOQCTL_DOCKER_SSH_KEY}|${ELOQCTL_CONTROL_SSH_KEY}|g" \ -e "s|\${ELOQKV_VERSION}|${ELOQKV_VERSION}|g" \ + -e "s|\${ELOQCTL_FEISHU_ROBOT_URL}|${ELOQCTL_FEISHU_ROBOT_URL}|g" \ "${source_topology}" > "${rendered_topology}" } @@ -95,41 +152,26 @@ prepare_control_node() { compose exec -T "${CONTROL_NODE_SERVICE}" bash -lc " set -eu install -d -m 700 -o eloq -g eloq /home/eloq/.ssh - install -m 600 -o eloq -g eloq '${CONTROL_REPO_ROOT}/tests/docker_ha/id_ed25519' '${ELOQCTL_CONTROL_SSH_KEY}' install -d -m 755 -o eloq -g eloq '${CONTROL_ELOQCTL_HOME}' - install -d -m 755 -o eloq -g eloq '${CONTROL_ELOQCTL_HOME}/bin' - cp '${CONTROL_REPO_ROOT}/target/debug/cluster_mgr' '${CONTROL_ELOQCTL}' - chmod 755 '${CONTROL_ELOQCTL}' - rm -f /usr/local/bin/eloqctl - printf '%s\n' '#!/bin/bash' \ - 'export HOME=/home/eloq' \ - 'export ELOQCTL_HOME=/home/eloq/.eloqctl' \ - 'exec /home/eloq/.eloqctl/bin/cluster_mgr "\$@"' \ - > /usr/local/bin/eloqctl - chmod 755 /usr/local/bin/eloqctl - rm -f '${CONTROL_ELOQCTL_HOME}/config' - ln -s '${CONTROL_REPO_ROOT}/src/cluster_mgr/config' '${CONTROL_ELOQCTL_HOME}/config' + install -m 600 -o eloq -g eloq '${CONTROL_REPO_ROOT}/tests/docker_ha/id_ed25519' '${ELOQCTL_CONTROL_SSH_KEY}' cat > /etc/profile.d/eloqctl.sh <<'EOF' export ELOQCTL_HOME='${CONTROL_ELOQCTL_HOME}' EOF grep -qxF 'export ELOQCTL_HOME=${ELOQCTL_HOME:-/home/eloq/.eloqctl}' /home/eloq/.bashrc || \ printf '\nexport ELOQCTL_HOME=${ELOQCTL_HOME:-/home/eloq/.eloqctl}\n' >> /home/eloq/.bashrc - chown -h eloq:eloq '${CONTROL_ELOQCTL_HOME}/config' - chown -R eloq:eloq '${CONTROL_ELOQCTL_HOME}' + chown -R eloq:eloq '${CONTROL_ELOQCTL_HOME}' /home/eloq/.ssh " } start_docker_env() { - ensure_dev_eloqctl ensure_ssh_key compose_down - echo "[docker] Build Ubuntu SSH containers" - COMPOSE_PROGRESS=plain compose build + ensure_docker_images echo "[docker] Start Docker HA network" - compose up -d >/dev/null + compose up -d --remove-orphans >/dev/null echo "[docker] Wait for MinIO" for _ in $(seq 1 60); do @@ -169,6 +211,8 @@ start_docker_env() { echo "[docker] Prepare control node ${CONTROL_NODE_SERVICE}" prepare_control_node + echo "[docker] Reset control-node runtime state" + reset_control_runtime_state_in_container } dump_failure_diagnostics() { @@ -179,7 +223,7 @@ dump_failure_diagnostics() { else echo "missing" fi - echo "---- eloqctl command logs ----" + echo "---- host eloqctl command logs ----" if [ -d "${ELOQCTL_HOME}/logs" ]; then ls -lt "${ELOQCTL_HOME}/logs" || true for file in "${ELOQCTL_HOME}"/logs/last-*.log; do @@ -188,6 +232,20 @@ dump_failure_diagnostics() { tail -80 "${file}" || true done fi + echo "---- control eloqctl command logs ----" + control_exec bash -lc ' + set +e + if [ -d "'"${CONTROL_ELOQCTL_HOME}"'/logs" ]; then + ls -lt "'"${CONTROL_ELOQCTL_HOME}"'/logs" + for file in "'"${CONTROL_ELOQCTL_HOME}"'/logs"/last-*.log; do + [ -f "${file}" ] || continue + echo "---- ${file} ----" + tail -80 "${file}" || true + done + else + echo "missing" + fi + ' || true echo "---- docker status ----" compose ps || true compose logs --no-color --tail=80 || true @@ -197,6 +255,11 @@ run_with_progress() { local timeout_seconds="$1" local log_file="$2" shift 2 + local control_log_file="" + if [ "${1:-}" = "--eloq-log" ]; then + control_log_file="$2" + shift 2 + fi : > "${log_file}" timeout --kill-after=10s "${timeout_seconds}s" "$@" > "${log_file}" 2>&1 & @@ -217,12 +280,12 @@ run_with_progress() { last_cmd_lines="${cmd_lines}" fi fi - if [ -f "${ELOQCTL_HOME}/logs/last-launch.log" ]; then + if [ -n "${control_log_file}" ]; then local eloq_lines - eloq_lines=$(wc -l < "${ELOQCTL_HOME}/logs/last-launch.log" 2>/dev/null || echo 0) + eloq_lines=$(control_exec bash -lc "wc -l < '${control_log_file}' 2>/dev/null || echo 0" 2>/dev/null | tr -d '\r' || echo 0) if [ "${eloq_lines}" -gt "${last_eloq_lines}" ]; then - echo " ---- new eloqctl log ----" - sed -n "$((last_eloq_lines + 1)),$((eloq_lines))p" "${ELOQCTL_HOME}/logs/last-launch.log" || true + echo " ---- new control eloqctl log (${control_log_file}) ----" + control_exec bash -lc "sed -n '$((last_eloq_lines + 1)),$((eloq_lines))p' '${control_log_file}' || true" || true last_eloq_lines="${eloq_lines}" fi fi diff --git a/tests/docker_ha/Dockerfile b/tests/docker_ha/Dockerfile index f15a3535..73a5dee4 100644 --- a/tests/docker_ha/Dockerfile +++ b/tests/docker_ha/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:24.04 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ + curl \ gdb \ gdbserver \ iproute2 \ diff --git a/tests/docker_ha/docker-compose.yaml b/tests/docker_ha/docker-compose.yaml index d5dd7f93..c97c9ef1 100644 --- a/tests/docker_ha/docker-compose.yaml +++ b/tests/docker_ha/docker-compose.yaml @@ -14,12 +14,12 @@ services: eloqctl-e2e-net: ipv4_address: 172.28.10.20 - eloq-node-1: + eloq-tx: build: context: . image: eloqctl-e2e-ubuntu:24.04 - container_name: eloqctl-e2e-node-1 - hostname: eloq-node-1 + container_name: eloqctl-e2e-tx + hostname: eloq-tx ports: - "127.0.0.1:2221:22" - "127.0.0.1:16371:6379" @@ -31,124 +31,77 @@ services: depends_on: - minio - eloq-node-2: + eloq-standby: image: eloqctl-e2e-ubuntu:24.04 - container_name: eloqctl-e2e-node-2 - hostname: eloq-node-2 + container_name: eloqctl-e2e-standby + hostname: eloq-standby ports: - "127.0.0.1:2222:22" - "127.0.0.1:16372:6379" - "127.0.0.1:26372:16380" depends_on: - - eloq-node-1 + - eloq-tx - minio networks: eloqctl-e2e-net: ipv4_address: 172.28.10.12 - eloq-node-3: + eloq-voter: image: eloqctl-e2e-ubuntu:24.04 - container_name: eloqctl-e2e-node-3 - hostname: eloq-node-3 + container_name: eloqctl-e2e-voter + hostname: eloq-voter ports: - "127.0.0.1:2223:22" - "127.0.0.1:16373:6379" - "127.0.0.1:26373:16380" depends_on: - - eloq-node-1 + - eloq-tx - minio networks: eloqctl-e2e-net: ipv4_address: 172.28.10.13 - eloq-node-4: + eloq-control: image: eloqctl-e2e-ubuntu:24.04 - container_name: eloqctl-e2e-node-4 - hostname: eloq-node-4 + container_name: eloqctl-e2e-control + hostname: eloq-control ports: - "127.0.0.1:2224:22" - "127.0.0.1:19010:9000" + - "127.0.0.1:19500:9500" + - "127.0.0.1:19093:9093" + - "127.0.0.1:13301:3301" + - "127.0.0.1:18080:8080" working_dir: /workspace/eloq_waiter volumes: - "../..:/workspace/eloq_waiter" depends_on: - - eloq-node-1 + - eloq-tx - minio networks: eloqctl-e2e-net: ipv4_address: 172.28.10.14 - stress-client: + stress-python: build: context: ../.. dockerfile: tests/docker_ha/Dockerfile.stress image: eloqctl-e2e-stress:latest - container_name: eloqctl-e2e-stress - hostname: stress-client + container_name: eloqctl-e2e-stress-python + hostname: stress-python working_dir: /workspace/eloq_waiter command: ["sleep", "infinity"] volumes: - "../..:/workspace/eloq_waiter" depends_on: - - eloq-node-1 - - eloq-node-2 - - eloq-node-3 - - eloq-node-4 + - eloq-tx + - eloq-standby + - eloq-voter + - eloq-control networks: eloqctl-e2e-net: ipv4_address: 172.28.10.30 - stress-client-2: - image: eloqctl-e2e-stress:latest - container_name: eloqctl-e2e-stress-2 - hostname: stress-client-2 - working_dir: /workspace/eloq_waiter - command: ["sleep", "infinity"] - volumes: - - "../..:/workspace/eloq_waiter" - depends_on: - - eloq-node-1 - - eloq-node-2 - - eloq-node-3 - - eloq-node-4 - networks: - eloqctl-e2e-net: - ipv4_address: 172.28.10.31 - - stress-client-3: - image: eloqctl-e2e-stress:latest - container_name: eloqctl-e2e-stress-3 - hostname: stress-client-3 - working_dir: /workspace/eloq_waiter - command: ["sleep", "infinity"] - volumes: - - "../..:/workspace/eloq_waiter" - depends_on: - - eloq-node-1 - - eloq-node-2 - - eloq-node-3 - - eloq-node-4 - networks: - eloqctl-e2e-net: - ipv4_address: 172.28.10.32 - - stress-client-4: - image: eloqctl-e2e-stress:latest - container_name: eloqctl-e2e-stress-4 - hostname: stress-client-4 - working_dir: /workspace/eloq_waiter - command: ["sleep", "infinity"] - volumes: - - "../..:/workspace/eloq_waiter" - depends_on: - - eloq-node-1 - - eloq-node-2 - - eloq-node-3 - - eloq-node-4 - networks: - eloqctl-e2e-net: - ipv4_address: 172.28.10.33 - stress-go: build: context: ../.. @@ -161,13 +114,13 @@ services: volumes: - "../..:/workspace/eloq_waiter" depends_on: - - eloq-node-1 - - eloq-node-2 - - eloq-node-3 - - eloq-node-4 + - eloq-tx + - eloq-standby + - eloq-voter + - eloq-control networks: eloqctl-e2e-net: - ipv4_address: 172.28.10.34 + ipv4_address: 172.28.10.31 stress-ts: build: @@ -181,13 +134,13 @@ services: volumes: - "../..:/workspace/eloq_waiter" depends_on: - - eloq-node-1 - - eloq-node-2 - - eloq-node-3 - - eloq-node-4 + - eloq-tx + - eloq-standby + - eloq-voter + - eloq-control networks: eloqctl-e2e-net: - ipv4_address: 172.28.10.35 + ipv4_address: 172.28.10.32 networks: eloqctl-e2e-net: diff --git a/tests/e2e/cmd_stress_go/cmd_stress_go b/tests/e2e/cmd_stress_go/cmd_stress_go new file mode 100755 index 00000000..058c2e8d Binary files /dev/null and b/tests/e2e/cmd_stress_go/cmd_stress_go differ diff --git a/tests/e2e/cmd_stress_go/main.go b/tests/e2e/cmd_stress_go/main.go index 408a3c16..3fbdf654 100644 --- a/tests/e2e/cmd_stress_go/main.go +++ b/tests/e2e/cmd_stress_go/main.go @@ -646,7 +646,6 @@ func main() { pref(logPrefix), strings.Join(addrs, ","), workers, inflight, duration, keyCount, commandSet, readFromReplicas) totalSlots := workers * inflight - // Cluster client for discovery clusterOpts := &redis.ClusterOptions{ Addrs: addrs, @@ -873,7 +872,6 @@ func main() { } logger.Printf("PASS") } - func pref(s string) string { if s == "" { return "" diff --git a/tests/e2e/cmd_stress_test.sh b/tests/e2e/cmd_stress_test.sh index 4e268fa6..8dfd57c4 100755 --- a/tests/e2e/cmd_stress_test.sh +++ b/tests/e2e/cmd_stress_test.sh @@ -2,14 +2,14 @@ # E2E multi-language command stress test for eloqkv cluster. # # Each language SDK runs inside its own Docker container (same subnet as cluster). -# - Python → stress-client (redis-py) +# - Python → stress-python (redis-py) # - Go → stress-go (go-redis/v9) # - TS → stress-ts (ioredis) # # Env overrides: -# STEPS=launch,eloqctl-mutate,py-stress,go-stress,ts-stress,remove +# STEPS=launch,monitor-update,eloqctl-mutate,py-stress,go-stress,ts-stress,remove # DURATION_SECONDS=300 WORKERS=8 INFLIGHT=4 KEY_COUNT=256 REPEAT=10 -# CMD_TIMEOUT=5 PROGRESS_INTERVAL=5 +# CMD_TIMEOUT=5 INFO_CMD_TIMEOUT=10 PROGRESS_INTERVAL=5 # TLS_ENABLED=1 SKIP_DEPS=1 set -euo pipefail @@ -20,7 +20,7 @@ source "${REPO_ROOT}/tests/docker_env.sh" CLUSTER="test-e2e" TOPO="${SCRIPT_DIR}/topology.generated.yaml" CONTROL_TOPO="${CONTROL_REPO_ROOT}/tests/e2e/topology.generated.yaml" -STEPS="${STEPS:-launch,eloqctl-mutate,py-stress,go-stress,ts-stress,remove}" +STEPS="${STEPS:-launch,monitor-update,eloqctl-mutate,py-stress,go-stress,ts-stress,remove}" DURATION="${DURATION_SECONDS:-300}" WORKERS="${WORKERS:-16}" @@ -28,6 +28,7 @@ INFLIGHT="${INFLIGHT:-4}" REPEAT="${REPEAT:-10}" KEY_COUNT="${KEY_COUNT:-256}" CMD_TIMEOUT="${CMD_TIMEOUT:-5}" +INFO_CMD_TIMEOUT="${INFO_CMD_TIMEOUT:-10}" PROGRESS_INTERVAL="${PROGRESS_INTERVAL:-5}" TLS_ENABLED="${TLS_ENABLED:-1}" SKIP_DEPS="${SKIP_DEPS:-1}" @@ -35,6 +36,14 @@ INFO_ONLY_WORKERS="${INFO_ONLY_WORKERS:-64}" INFO_ONLY_INFLIGHT="${INFO_ONLY_INFLIGHT:-16}" INFO_ONLY_REPEAT="${INFO_ONLY_REPEAT:-50}" INFO_ONLY_DURATION="${INFO_ONLY_DURATION_SECONDS:-300}" +TS_INFO_ONLY_WORKERS="${TS_INFO_ONLY_WORKERS:-16}" +TS_INFO_ONLY_INFLIGHT="${TS_INFO_ONLY_INFLIGHT:-8}" +TS_INFO_ONLY_REPEAT="${TS_INFO_ONLY_REPEAT:-50}" +TS_INFO_ONLY_DURATION="${TS_INFO_ONLY_DURATION_SECONDS:-30}" +GRAFANA_UPDATE_URL="${GRAFANA_UPDATE_URL:-https://dl.grafana.com/grafana/release/13.0.1+security-01/grafana_13.0.1+security-01_25720641773_linux_amd64.tar.gz}" +GRAFANA_HTTP_URL="${GRAFANA_HTTP_URL:-http://172.28.10.14:3301}" +ALERTMANAGER_HTTP_URL="${ALERTMANAGER_HTTP_URL:-http://172.28.10.14:9093}" +ALERTMANAGER_WEBHOOK_ADAPTER_HTTP_URL="${ALERTMANAGER_WEBHOOK_ADAPTER_HTTP_URL:-http://172.28.10.14:8080}" PASSWD="testpass" N1="172.28.10.11" @@ -50,28 +59,108 @@ TLS_FLAG="" [ "${TLS_ENABLED}" = "1" ] && TLS_FLAG="--tls" control_eloqctl_cmd() { - control_exec env ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" "${CONTROL_ELOQCTL}" "$@" + control_ssh_cmd "$@" +} + +control_ssh_exec_string() { + local remote_cmd + printf -v remote_cmd '%q ' \ + env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" "${CONTROL_ELOQCTL}" "$@" + printf 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o BatchMode=yes -o ConnectTimeout=3 -i %q eloq@127.0.0.1 -p 2224 %q' \ + "${ELOQCTL_DOCKER_SSH_KEY}" \ + "bash -lc $(printf '%q' "${remote_cmd}")" } run_control_eloqctl_with_progress() { local timeout_seconds="$1" local log_file="$2" shift 2 - run_with_progress "${timeout_seconds}" "${log_file}" \ - docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ - "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ - "${CONTROL_ELOQCTL}" "$@" \ + local subcommand="$1" + local control_log_file="${CONTROL_ELOQCTL_HOME}/logs/last-${subcommand}.log" + run_with_progress "${timeout_seconds}" "${log_file}" --eloq-log "${control_log_file}" \ + bash -lc "$(control_ssh_exec_string "$@")" \ || { dump_failure_diagnostics "${log_file}"; return 1; } } wait_cluster_ready() { - run_with_progress 240 "${SCRIPT_DIR}/cmd-stress-status.log" \ - docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ - "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ - "${CONTROL_ELOQCTL}" status "${CLUSTER}" --wait 180 >/dev/null 2>&1 \ + run_with_progress 240 "${SCRIPT_DIR}/cmd-stress-status.log" --eloq-log "${CONTROL_ELOQCTL_HOME}/logs/last-status.log" \ + bash -lc "$(control_ssh_exec_string status "${CLUSTER}" --wait 180)" >/dev/null 2>&1 \ || { echo "FAIL: cluster not healthy"; return 1; } } +wait_monitor_ready() { + local url="${1:-${GRAFANA_HTTP_URL}}" + for _ in $(seq 1 60); do + if control_exec curl -fsS "${url}/api/health" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "FAIL: monitor endpoint not healthy at ${url}" + return 1 +} + +wait_alertmanager_ready() { + local url="${1:-${ALERTMANAGER_HTTP_URL}}" + for _ in $(seq 1 60); do + if control_exec curl -fsS "${url}/-/healthy" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "FAIL: alertmanager endpoint not healthy at ${url}" + return 1 +} + +wait_alertmanager_webhook_adapter_ready() { + local url="${1:-${ALERTMANAGER_WEBHOOK_ADAPTER_HTTP_URL}}" + for _ in $(seq 1 60); do + if control_exec curl -fsS "${url}" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "FAIL: alertmanager-webhook-adapter endpoint not healthy at ${url}" + return 1 +} + +assert_cluster_registered() { + control_eloqctl_cmd list | grep -Eq "[[:space:]]${CLUSTER}[[:space:]]" \ + || { echo "FAIL: eloqctl list does not contain cluster ${CLUSTER}"; return 1; } +} + +refresh_monitor_status() { + run_control_eloqctl_with_progress 180 "${SCRIPT_DIR}/monitor-status.log" \ + monitor status --cluster "${CLUSTER}" >/dev/null \ + || return 1 +} + +cluster_has_monitor_service() { + local service="$1" + grep -Eq "\\|[[:space:]]*${service}[[:space:]]*\\|" "${SCRIPT_DIR}/monitor-status.log" +} + +assert_monitor_version() { + local expected_major="$1" + local url="${2:-${GRAFANA_HTTP_URL}}" + local health + health=$(control_exec curl -fsS "${url}/api/health") \ + || { echo "FAIL: failed to query monitor health from ${url}"; return 1; } + echo "${health}" | grep -Eq "\"version\"[[:space:]]*:[[:space:]]*\"${expected_major}\\." \ + || { + echo "FAIL: expected Grafana ${expected_major}.x, got: ${health}" + return 1 + } +} + +assert_export_contains() { + local expected="$1" + local export_file="/tmp/${CLUSTER}-export.yaml" + control_eloqctl_cmd export "${CLUSTER}" --output "${export_file}" >/dev/null + control_exec grep -F "${expected}" "${export_file}" >/dev/null \ + || { echo "FAIL: export does not contain expected text: ${expected}"; return 1; } +} + cleanup() { if [ "${KEEP_LOGS:-0}" != "1" ]; then rm -f "${SCRIPT_DIR}/cmd-stress-"*.log "${SCRIPT_DIR}/launch-cmd-stress.log" "${TOPO}" @@ -103,7 +192,7 @@ step() { discover_master() { echo " discovering cluster topology ..." local nodes_info - nodes_info=$(compose exec -T stress-client python3 -c " + nodes_info=$(docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-python python3 -c " import ssl TLS={'ssl':True,'ssl_cert_reqs':ssl.CERT_NONE,'ssl_check_hostname':False} from redis import Redis @@ -139,24 +228,56 @@ do_launch() { if [ "${SKIP_DEPS}" = "1" ]; then launch_args+=(--skip-deps) fi - render_topology_for_control "${SCRIPT_DIR}/topology.yaml" "${TOPO}" start_docker_env + (cd "${REPO_ROOT}" && bash tests/install_control_eloqctl.sh) + render_topology_for_control "${SCRIPT_DIR}/topology.yaml" "${TOPO}" + control_exec test -f "${CONTROL_TOPO}" \ + || { echo "FAIL: control topology not found at ${CONTROL_TOPO}"; return 1; } control_eloqctl_cmd stop "${CLUSTER}" --all --force >/dev/null 2>&1 || true control_eloqctl_cmd remove "${CLUSTER}" --force >/dev/null 2>&1 || true - run_with_progress 420 "${SCRIPT_DIR}/launch-cmd-stress.log" \ - docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ - "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ - "${CONTROL_ELOQCTL}" launch "${launch_args[@]}" "${CONTROL_TOPO}" \ + run_with_progress 420 "${SCRIPT_DIR}/launch-cmd-stress.log" --eloq-log "${CONTROL_ELOQCTL_HOME}/logs/last-launch.log" \ + bash -lc "$(control_ssh_exec_string launch "${launch_args[@]}" "${CONTROL_TOPO}")" \ || { dump_failure_diagnostics "${SCRIPT_DIR}/launch-cmd-stress.log"; return 1; } - run_with_progress 240 "${SCRIPT_DIR}/launch-cmd-stress.log" \ - docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T -u eloq \ - "${CONTROL_NODE_SERVICE}" env HOME=/home/eloq ELOQCTL_HOME="${CONTROL_ELOQCTL_HOME}" \ - "${CONTROL_ELOQCTL}" status "${CLUSTER}" --wait 180 >/dev/null 2>&1 \ + run_with_progress 240 "${SCRIPT_DIR}/launch-cmd-stress.log" --eloq-log "${CONTROL_ELOQCTL_HOME}/logs/last-status.log" \ + bash -lc "$(control_ssh_exec_string status "${CLUSTER}" --wait 180)" >/dev/null 2>&1 \ || { echo "FAIL: cluster not healthy after launch"; return 1; } + assert_cluster_registered || return 1 + refresh_monitor_status || { echo "FAIL: monitor status failed after launch"; return 1; } + if cluster_has_monitor_service "grafana"; then + wait_monitor_ready || return 1 + fi + if cluster_has_monitor_service "alertmanager"; then + wait_alertmanager_ready || return 1 + fi + if cluster_has_monitor_service "alertmanager_webhook_adapter"; then + wait_alertmanager_webhook_adapter_ready || return 1 + fi + if cluster_has_monitor_service "grafana"; then + assert_monitor_version 9 || return 1 + fi echo " cluster ready" discover_master || return 1 } +do_monitor_update() { + echo "=== Monitor Grafana update check ===" + assert_export_contains "grafana-9.3.6.linux-amd64.tar.gz" || return 1 + assert_cluster_registered || return 1 + run_control_eloqctl_with_progress 420 "${SCRIPT_DIR}/cmd-stress-monitor-update.log" \ + monitor update --cluster "${CLUSTER}" --component grafana --url "${GRAFANA_UPDATE_URL}" \ + || return 1 + refresh_monitor_status || return 1 + if ! cluster_has_monitor_service "grafana"; then + echo "FAIL: monitor status does not report grafana after update" + return 1 + fi + wait_monitor_ready || return 1 + assert_monitor_version 13 || return 1 + wait_cluster_ready || return 1 + assert_export_contains "${GRAFANA_UPDATE_URL}" || return 1 + echo " grafana update verified" +} + do_eloqctl_mutate() { local original_master original_replica local original_master_host original_master_port @@ -211,7 +332,7 @@ do_eloqctl_mutate() { echo " eloqctl mutations verified" } -# ── Python stress (inside stress-client container) ── +# ── Python stress (inside stress-python container) ── do_py_stress() { discover_master || return 1 echo "=== Python command stress (redis-py) ===" @@ -219,7 +340,7 @@ do_py_stress() { if [ -n "${REPLICA}" ]; then py_startup_args+=(--startup-node "${REPLICA}") fi - compose exec -T stress-client python3 -u tests/e2e/cmd_stress_py/main.py \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-python python3 -u tests/e2e/cmd_stress_py/main.py \ "${py_startup_args[@]}" \ --password "${PASSWD}" --cmd-timeout "${CMD_TIMEOUT}" \ --progress-interval "${PROGRESS_INTERVAL}" --key-count "${KEY_COUNT}" \ @@ -229,9 +350,9 @@ do_py_stress() { [ ${py_status} -eq 0 ] || return ${py_status} echo "=== Python INFO burst (redis-py) ===" - compose exec -T stress-client python3 -u tests/e2e/cmd_stress_py/main.py \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-python python3 -u tests/e2e/cmd_stress_py/main.py \ "${py_startup_args[@]}" \ - --password "${PASSWD}" --cmd-timeout "${CMD_TIMEOUT}" \ + --password "${PASSWD}" --cmd-timeout "${INFO_CMD_TIMEOUT}" \ --progress-interval "${PROGRESS_INTERVAL}" --key-count "${KEY_COUNT}" \ --workers "${INFO_ONLY_WORKERS}" --inflight "${INFO_ONLY_INFLIGHT}" \ --duration "${INFO_ONLY_DURATION}" --repeat "${INFO_ONLY_REPEAT}" \ @@ -244,9 +365,9 @@ do_py_stress() { do_go_stress() { echo "=== Go command stress (go-redis/v9) ===" echo " installing Go deps ..." - compose exec -T stress-go bash -c \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-go bash -c \ 'cd tests/e2e/cmd_stress_go && go mod download' 2>&1 || true - compose exec -T stress-go bash -c \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-go bash -c \ "cd tests/e2e/cmd_stress_go && go run . \ --startup-nodes '${STARTUP_NODES}' \ --password '${PASSWD}' \ @@ -263,7 +384,7 @@ do_go_stress() { [ ${go_status} -eq 0 ] || return ${go_status} echo "=== Go INFO burst (go-redis/v9) ===" - compose exec -T stress-go bash -c \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-go bash -c \ "cd tests/e2e/cmd_stress_go && go run . \ --startup-nodes '${STARTUP_NODES}' \ --password '${PASSWD}' \ @@ -273,7 +394,7 @@ do_go_stress() { --repeat ${INFO_ONLY_REPEAT} \ --progress-interval ${PROGRESS_INTERVAL}s \ --key-count ${KEY_COUNT} \ - --cmd-timeout ${CMD_TIMEOUT}s \ + --cmd-timeout ${INFO_CMD_TIMEOUT}s \ --command-set info-only \ $([ "${TLS_ENABLED}" = "1" ] && echo '--tls-insecure')" \ 2>&1 | tee "${SCRIPT_DIR}/cmd-stress-go-info.log" @@ -284,9 +405,9 @@ do_go_stress() { do_ts_stress() { echo "=== TypeScript command stress (ioredis) ===" echo " installing npm deps ..." - compose exec -T stress-ts bash -c \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-ts bash -c \ 'cd tests/e2e/cmd_stress_ts && npm install --silent' 2>&1 || true - compose exec -T stress-ts bash -c \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-ts bash -c \ "cd tests/e2e/cmd_stress_ts && npx tsx main.ts \ --startup-nodes '${STARTUP_NODES}' \ --password '${PASSWD}' \ @@ -304,17 +425,17 @@ do_ts_stress() { [ ${ts_status} -eq 0 ] || return ${ts_status} echo "=== TypeScript INFO burst (ioredis) ===" - compose exec -T stress-ts bash -c \ + docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-ts bash -c \ "cd tests/e2e/cmd_stress_ts && npx tsx main.ts \ --startup-nodes '${STARTUP_NODES}' \ --password '${PASSWD}' \ - --workers ${INFO_ONLY_WORKERS} \ - --inflight ${INFO_ONLY_INFLIGHT} \ - --duration ${INFO_ONLY_DURATION} \ - --repeat ${INFO_ONLY_REPEAT} \ + --workers ${TS_INFO_ONLY_WORKERS} \ + --inflight ${TS_INFO_ONLY_INFLIGHT} \ + --duration ${TS_INFO_ONLY_DURATION} \ + --repeat ${TS_INFO_ONLY_REPEAT} \ --progress-interval ${PROGRESS_INTERVAL} \ --key-count ${KEY_COUNT} \ - --cmd-timeout ${CMD_TIMEOUT} \ + --cmd-timeout ${INFO_CMD_TIMEOUT} \ --command-set info-only \ --read-from-replicas 'false' \ --tls-insecure '${TLS_ENABLED}'" \ @@ -332,6 +453,7 @@ do_remove() { } step launch do_launch +step monitor-update do_monitor_update step eloqctl-mutate do_eloqctl_mutate step py-stress do_py_stress step go-stress do_go_stress diff --git a/tests/e2e/cmd_stress_ts/main.ts b/tests/e2e/cmd_stress_ts/main.ts index b64405bd..179ca0d6 100644 --- a/tests/e2e/cmd_stress_ts/main.ts +++ b/tests/e2e/cmd_stress_ts/main.ts @@ -70,6 +70,24 @@ function mkClient(addr: string): Redis { }); } +function mkClusterClient(startup: string[], readFromReplicas: boolean): Cluster { + return new Cluster(startup.map(addr => { + const [host, portStr] = addr.split(":"); + return { host, port: parseInt(portStr || "6379") }; + }), { + scaleReads: readFromReplicas ? "slave" : "master", + slotsRefreshTimeout: CMD_TIMEOUT, + redisOptions: { + password, + connectTimeout: CMD_TIMEOUT, + commandTimeout: CMD_TIMEOUT, + retryStrategy: () => null, + ...(TLS_INSECURE ? { tls: { rejectUnauthorized: false } as any } : {}), + }, + clusterRetryStrategy: () => null, + }); +} + // --------------------------------------------------------------------------- // Command test definitions // Each entry: [name, fn(client, idx) => Promise] @@ -301,21 +319,7 @@ async function main() { console.log(P(`nodes=${startupNodes} workers=${WORKERS} inflight=${INFLIGHT} duration=${DURATION}s key_count=${KEY_COUNT} command_set=${COMMAND_SET} read_from_replicas=${READ_FROM_REPLICAS}`)); // Cluster discovery - const cluster = new Cluster(startupNodes.map(addr => { - const [host, portStr] = addr.split(":"); - return { host, port: parseInt(portStr || "6379") }; - }), { - scaleReads: READ_FROM_REPLICAS ? "slave" : "master", - slotsRefreshTimeout: CMD_TIMEOUT, - redisOptions: { - password, - connectTimeout: CMD_TIMEOUT, - commandTimeout: CMD_TIMEOUT, - retryStrategy: () => null, - ...(TLS_INSECURE ? { tls: { rejectUnauthorized: false } as any } : {}), - }, - clusterRetryStrategy: () => null, - }); + const cluster = mkClusterClient(startupNodes, READ_FROM_REPLICAS); await cluster.ping(); console.log(P("cluster ping OK")); @@ -397,12 +401,44 @@ async function main() { const clusterStats = new CmdStats(); const controller = new AbortController(); + const standaloneClients: Redis[] = []; + const clusterClients: Cluster[] = []; const workers: Promise[] = []; - for (let w = 0; w < totalStandaloneSlots; w++) { - workers.push(stressWorker(master, SELECTED_CMD_TESTS, standaloneStats, controller.signal, w, KEY_COUNT, REPEAT)); + for (let w = 0; w < nStandalone; w++) { + const client = mkClient(masterAddr); + standaloneClients.push(client); + const baseSlot = w * INFLIGHT; + for (let lane = 0; lane < INFLIGHT; lane++) { + workers.push( + stressWorker( + client, + SELECTED_CMD_TESTS, + standaloneStats, + controller.signal, + baseSlot + lane, + KEY_COUNT, + REPEAT, + ), + ); + } } - for (let w = 0; w < totalClusterSlots; w++) { - workers.push(stressWorker(cluster, SELECTED_CMD_TESTS, clusterStats, controller.signal, w + totalStandaloneSlots, KEY_COUNT, REPEAT)); + for (let w = 0; w < nCluster; w++) { + const client = mkClusterClient(startupNodes, READ_FROM_REPLICAS); + clusterClients.push(client); + const baseSlot = w * INFLIGHT; + for (let lane = 0; lane < INFLIGHT; lane++) { + workers.push( + stressWorker( + client, + SELECTED_CMD_TESTS, + clusterStats, + controller.signal, + totalStandaloneSlots + baseSlot + lane, + KEY_COUNT, + REPEAT, + ), + ); + } } const start = Date.now(); @@ -432,6 +468,8 @@ async function main() { clearInterval(progressInterval); clearInterval(checkStop); + for (const client of standaloneClients) client.disconnect(); + for (const client of clusterClients) client.disconnect(); master.disconnect(); if (replica) replica.disconnect(); cluster.disconnect(); diff --git a/tests/e2e/devctl.sh b/tests/e2e/devctl.sh new file mode 100755 index 00000000..f80c6cee --- /dev/null +++ b/tests/e2e/devctl.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +source "${REPO_ROOT}/tests/docker_env.sh" + +CONTROL_TOPO_TEMPLATE="tests/e2e/topology.yaml" +CONTROL_TOPO="/home/eloq/topology.generated.yaml" +DEFAULT_CLUSTER="test-e2e" + +usage() { + cat <<'EOF' +Usage: tests/e2e/devctl.sh [args] + +Commands: + build Build eloqctl locally + env-up Start the Docker E2E environment + env-down Stop and remove the Docker E2E environment + install-control Copy the current eloqctl build into the control node + control-shell SSH into the control node + render-topology Render topology.yaml inside the control node + launch Launch the default E2E cluster from the control node + status Show cluster and monitor status + grafana-update [url] Upgrade Grafana with monitor update + export-topology Export cluster topology from the control node + stress [steps] Run cmd_stress_test.sh via devctl; default steps are unchanged + full build -> env-up -> install-control -> render-topology -> launch + +Environment: + CLUSTER_NAME Default: test-e2e + ELOQKV_VERSION Override the EloqKV version during topology rendering +EOF +} + +cluster_name() { + echo "${CLUSTER_NAME:-${DEFAULT_CLUSTER}}" +} + +build_eloqctl() { + (cd "${REPO_ROOT}" && cargo build -p cluster_mgr) +} + +install_control() { + (cd "${REPO_ROOT}" && bash tests/install_control_eloqctl.sh) +} + +render_control_topology() { + control_exec bash -lc " + cd '${CONTROL_REPO_ROOT}' + source tests/docker_env.sh + render_topology_for_control '${CONTROL_TOPO_TEMPLATE}' '${CONTROL_TOPO}' + ls -l '${CONTROL_TOPO}' + " +} + +launch_cluster() { + local cluster + cluster="$(cluster_name)" + control_exec bash -lc " + eloqctl stop '${cluster}' --all --force >/dev/null 2>&1 || true + eloqctl remove '${cluster}' --force >/dev/null 2>&1 || true + eloqctl launch --skip-deps '${CONTROL_TOPO}' + " +} + +show_status() { + local cluster + cluster="$(cluster_name)" + control_exec bash -lc " + eloqctl status '${cluster}' --wait 180 + eloqctl monitor status --cluster '${cluster}' + " +} + +grafana_update() { + local cluster url + cluster="$(cluster_name)" + url="${1:-https://dl.grafana.com/grafana/release/13.0.1+security-01/grafana_13.0.1+security-01_25720641773_linux_amd64.tar.gz}" + control_exec bash -lc " + eloqctl monitor update --cluster '${cluster}' --component grafana --url '${url}' + " +} + +export_topology() { + local cluster + cluster="$(cluster_name)" + control_exec bash -lc " + eloqctl export '${cluster}' --output /home/eloq/${cluster}-export.yaml + ls -l /home/eloq/${cluster}-export.yaml + " +} + +stress_run() { + local steps_arg="${1:-}" + if [ -n "${steps_arg}" ]; then + (cd "${REPO_ROOT}" && STEPS="${steps_arg}" bash tests/e2e/cmd_stress_test.sh) + else + (cd "${REPO_ROOT}" && bash tests/e2e/cmd_stress_test.sh) + fi +} + +control_shell() { + exec ssh -i "${DOCKER_E2E_DIR}/id_ed25519" -p 2224 eloq@127.0.0.1 +} + +cmd="${1:-}" +shift || true + +case "${cmd}" in + build) + build_eloqctl + ;; + env-up) + start_docker_env + ;; + env-down) + compose_down + ;; + install-control) + install_control + ;; + control-shell) + control_shell + ;; + render-topology) + render_control_topology + ;; + launch) + launch_cluster + ;; + status) + show_status + ;; + grafana-update) + grafana_update "${1:-}" + ;; + export-topology) + export_topology + ;; + stress) + stress_run "${1:-}" + ;; + full) + build_eloqctl + start_docker_env + install_control + render_control_topology + launch_cluster + ;; + ""|-h|--help|help) + usage + ;; + *) + echo "unknown command: ${cmd}" >&2 + usage >&2 + exit 1 + ;; +esac diff --git a/tests/e2e/topology.yaml b/tests/e2e/topology.yaml index 3986c25c..8f06735a 100644 --- a/tests/e2e/topology.yaml +++ b/tests/e2e/topology.yaml @@ -40,3 +40,19 @@ deployment: bucket_name: "eloqservice" bucket_prefix: "store" endpoint: "http://172.28.10.20:9000" + monitor: + data_dir: "" + eloq_metrics: + path: "/eloq_metrics" + port: 18081 + prometheus: + download_url: "https://github.com/prometheus/prometheus/releases/download/v2.42.0/prometheus-2.42.0.linux-amd64.tar.gz" + port: 9500 + host: "172.28.10.14" + grafana: + download_url: "https://dl.grafana.com/oss/release/grafana-9.3.6.linux-amd64.tar.gz" + port: 3301 + host: "172.28.10.14" + node_exporter: + url: "https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-amd64.tar.gz" + port: 9200 diff --git a/tests/install_control_eloqctl.sh b/tests/install_control_eloqctl.sh index d6fd869a..c2c22d8b 100755 --- a/tests/install_control_eloqctl.sh +++ b/tests/install_control_eloqctl.sh @@ -44,6 +44,11 @@ compose exec -T "${CONTROL_NODE_SERVICE}" bash -lc " install -d -m 700 -o eloq -g eloq /home/eloq/.ssh install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}' install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}/bin' + install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}/db' + install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}/download' + install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}/logs' + install -d -m 755 -o eloq -g eloq '${INSTALL_ROOT}/upload' + rm -f '${INSTALL_BIN}' rm -rf '${INSTALL_ROOT}/config' "