From fb54135249ba062b8a88f045d130996c5937ea51 Mon Sep 17 00:00:00 2001 From: Teg79 Date: Thu, 19 Nov 2026 20:51:15 +0100 Subject: [PATCH 1/2] feat(observability): the Proxmox node's CPU, memory and storages in Grafana, with alerts, from a read-only token Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182yxsfwRowwnxCfgrQwjnA --- CHANGELOG.md | 12 ++ internal/bootstrap/orchestrator.go | 5 +- internal/bootstrap/public_test.go | 53 ++++++++ internal/bootstrap/stacks.go | 74 ++++++++++- internal/cli/doctor.go | 8 +- internal/credentials/credentials.go | 7 + internal/doctor/nodemetrics.go | 53 ++++++++ internal/doctor/nodemetrics_test.go | 49 +++++++ internal/platform/catalog.go | 6 + platform/stacks/observability/compose.yaml | 21 +++ .../grafana/dashboards/urbex-resources.json | 123 ++++++++++++++++++ .../provisioning/alerting/urbex-resources.yml | 88 +++++++++++++ platform/stacks/observability/prometheus.yml | 22 ++++ 13 files changed, 515 insertions(+), 6 deletions(-) create mode 100644 internal/doctor/nodemetrics.go create mode 100644 internal/doctor/nodemetrics_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 31292a1..5008a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,18 @@ workflow refuses a tag whose version has no entry here. ## [Unreleased] +### Added + +- **The Proxmox node in Grafana** (#94): with an optional read-only token + (`proxmoxMonitorToken`, role `PVEAuditor`), `urbex bootstrap` runs + `pve-exporter` in the observability Stack. The *Urbex resources* + dashboard then shows the node's CPU, memory (guests outside urbex and + ZFS's ARC included, as Proxmox counts it) and each storage's use, and + Grafana alerts when the node's memory stays above 90% or a storage + passes 85% / 95%. Only the node and storage series are kept, never other + guests'. `urbex doctor` checks the token, or says the node metrics are + off without it. + ## [0.1.0] - 2026-11-17 The first preview release: urbex as binaries for Linux and macOS (amd64, diff --git a/internal/bootstrap/orchestrator.go b/internal/bootstrap/orchestrator.go index 353183e..c37c468 100644 --- a/internal/bootstrap/orchestrator.go +++ b/internal/bootstrap/orchestrator.go @@ -197,7 +197,10 @@ func (o *Orchestrator) Run(ctx context.Context, opts Options) error { // The platform services Komodo deploys from the GitOps repo: their // folders go in before setupPlatform pushes it. o.Progress.Begin(PhaseGitOps) - stacks := o.platformStacks(tunnelToken) + stacks, err := o.platformStacks(tunnelToken) + if err != nil { + return err + } if err := o.writePlatformStacks(ctx, opts.GitOpsRepoPath, stacks); err != nil { return err } diff --git a/internal/bootstrap/public_test.go b/internal/bootstrap/public_test.go index 7861eb8..cdefddd 100644 --- a/internal/bootstrap/public_test.go +++ b/internal/bootstrap/public_test.go @@ -463,3 +463,56 @@ func TestBaseServicesAreInTheCatalog(t *testing.T) { } } } + +// The node's metrics (urbforge/urbex-cli#94): with the read-only token, +// pve-exporter's profile is on, reaching the node by its address, the +// token's parts in the Stack's secrets; without it, the profile is +// explicitly off. +func TestRun_NodeMetricsWithTheMonitoringToken(t *testing.T) { + dir := t.TempDir() + o, _, fp := newRunOrchestrator(t, &fakeProxmox{}) + o.Config.Proxmox.APIURL = "https://192.0.2.10:8006/api2/json" + o.Credentials.ProxmoxMonitorToken = "urbex-monitor@pve!metrics=00000000-1111-2222-3333-444444444444" + if err := o.Run(context.Background(), Options{GitOpsRepoPath: dir}); err != nil { + t.Fatalf("Run: %v", err) + } + if env := fp.stacks["urbex-observability"].Environment; env != "COMPOSE_PROFILES=pve\nURBEX_PVE_NODE_IP=192.0.2.10\n" { + t.Errorf("observability environment = %q", env) + } + secrets, _ := os.ReadFile(filepath.Join(dir, "platform/observability/secrets.sops.env")) + for _, want := range []string{"PVE_USER=urbex-monitor@pve", "PVE_TOKEN_NAME=metrics", "PVE_TOKEN_VALUE=00000000-1111-2222-3333-444444444444"} { + if !strings.Contains(string(secrets), want) { + t.Errorf("secrets lack %s:\n%s", want, secrets) + } + } + config, _ := os.ReadFile(filepath.Join(dir, "platform/observability/config.env")) + if strings.Contains(string(config), "00000000-1111") || !strings.Contains(string(config), "PVE_VERIFY_SSL=false") { + t.Errorf("config.env:\n%s", config) + } + + o2, _, fp2 := newRunOrchestrator(t, &fakeProxmox{}) + if err := o2.Run(context.Background(), Options{GitOpsRepoPath: t.TempDir()}); err != nil { + t.Fatalf("Run without the token: %v", err) + } + if env := fp2.stacks["urbex-observability"].Environment; env != "COMPOSE_PROFILES=\nURBEX_PVE_NODE_IP=\n" { + t.Errorf("without the token, environment = %q", env) + } + + o3, _, _ := newRunOrchestrator(t, &fakeProxmox{}) + o3.Credentials.ProxmoxMonitorToken = "not-a-token" + if err := o3.Run(context.Background(), Options{GitOpsRepoPath: t.TempDir()}); err == nil || !strings.Contains(err.Error(), "proxmoxMonitorToken") { + t.Errorf("a malformed token: %v", err) + } +} + +func TestSplitProxmoxToken(t *testing.T) { + u, n, v, err := SplitProxmoxToken("urbex@pve!urbex=a-b=c") + if err != nil || u != "urbex@pve" || n != "urbex" || v != "a-b=c" { + t.Errorf("got %q %q %q %v", u, n, v, err) + } + for _, bad := range []string{"", "urbex@pve!urbex", "urbex!x=y", "@pve!x=y", "u@pve!=y", "u@pve!x="} { + if _, _, _, err := SplitProxmoxToken(bad); err == nil { + t.Errorf("%q accepted", bad) + } + } +} diff --git a/internal/bootstrap/stacks.go b/internal/bootstrap/stacks.go index c206f57..8e49022 100644 --- a/internal/bootstrap/stacks.go +++ b/internal/bootstrap/stacks.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "io/fs" + "net" + "net/url" "os" "path" "path/filepath" @@ -34,6 +36,9 @@ type PlatformStack struct { Name, Dir string Config map[string]string Secrets map[string]string + // Environment is the Stack's .env, for compose itself (profiles, + // interpolation), set in Komodo; "" leaves it empty. + Environment string } // ComposeWrapper is the command every Stack deployed from the GitOps repo @@ -52,7 +57,7 @@ const PlatformStacksDir = "platform" // platformStacks are the platform services deployed from the GitOps // repo: Keycloak, the observability stack and, with Cloudflare, the // tunnel, whose connector token is tunnelToken. -func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack { +func (o *Orchestrator) platformStacks(tunnelToken string) ([]PlatformStack, error) { // Keycloak runs in production mode on its own PostgreSQL, in the same // Stack (urbex-cli#36). Both containers read config.env and the // secrets: KC_* configure Keycloak, POSTGRES_* the database. @@ -89,14 +94,74 @@ func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack { "KC_DB_PASSWORD": o.Credentials.KeycloakDBPassword, "POSTGRES_PASSWORD": o.Credentials.KeycloakDBPassword, }}, - {Name: Hostname("observability"), Dir: "observability", Config: map[string]string{}, - Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword}}, } + observability, err := o.observabilityStack() + if err != nil { + return nil, err + } + stacks = append(stacks, observability) if o.Config.Cloudflare.Enabled() { stacks = append(stacks, PlatformStack{Name: Hostname(TunnelService.Name), Dir: "tunnel", Config: map[string]string{}, Secrets: map[string]string{"TUNNEL_TOKEN": tunnelToken}}) } - return stacks + return stacks, nil +} + +// observabilityStack is Prometheus, Loki and Grafana; with the node's +// read-only token, also pve-exporter for the node's metrics +// (urbforge/urbex-cli#94). +func (o *Orchestrator) observabilityStack() (PlatformStack, error) { + st := PlatformStack{Name: Hostname("observability"), Dir: "observability", Config: map[string]string{}, + Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword}, + // Written even without the token, so that removing it turns the + // exporter off. + Environment: "COMPOSE_PROFILES=\nURBEX_PVE_NODE_IP=\n"} + if o.Credentials.ProxmoxMonitorToken == "" { + return st, nil + } + user, name, value, err := SplitProxmoxToken(o.Credentials.ProxmoxMonitorToken) + if err != nil { + return st, fmt.Errorf("proxmoxMonitorToken: %w", err) + } + ip, err := nodeIP(o.Config.Proxmox.APIURL) + if err != nil { + return st, err + } + st.Secrets["PVE_USER"], st.Secrets["PVE_TOKEN_NAME"], st.Secrets["PVE_TOKEN_VALUE"] = user, name, value + // The exporter reaches the node as pve-node, so its certificate's name + // never matches; the node's is self-signed anyway, and the token only + // reads. + st.Config["PVE_VERIFY_SSL"] = "false" + st.Environment = "COMPOSE_PROFILES=pve\nURBEX_PVE_NODE_IP=" + ip + "\n" + return st, nil +} + +// SplitProxmoxToken splits an API token, "user@realm!name=secret", into +// the user, the token's name and its secret. +func SplitProxmoxToken(token string) (user, name, value string, err error) { + id, value, ok := strings.Cut(token, "=") + user, name, ok2 := strings.Cut(id, "!") + if !ok || !ok2 || user == "" || name == "" || value == "" || strings.Index(user, "@") < 1 { + return "", "", "", fmt.Errorf("not a Proxmox API token: expected user@realm!name=secret") + } + return user, name, value, nil +} + +// nodeIP is the address of the Proxmox API's host, resolved if it is a +// name. +func nodeIP(apiURL string) (string, error) { + u, err := url.Parse(apiURL) + if err != nil || u.Hostname() == "" { + return "", fmt.Errorf("proxmox.apiUrl %q has no host", apiURL) + } + if ip := net.ParseIP(u.Hostname()); ip != nil { + return ip.String(), nil + } + ips, err := net.LookupIP(u.Hostname()) + if err != nil || len(ips) == 0 { + return "", fmt.Errorf("resolving the Proxmox node %s: %v", u.Hostname(), err) + } + return ips[0].String(), nil } // Keycloak's database and its owner, in the Keycloak Stack's PostgreSQL. @@ -208,6 +273,7 @@ func (o *Orchestrator) platformStackDeploys(stacks []PlatformStack) ([]stackDepl {Path: "secrets.sops.env", Requires: "Redeploy"}, }, ComposeCmdWrapper: ComposeWrapper, + Environment: st.Environment, }}) } return deploys, nil diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 2582353..686685f 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -31,7 +31,8 @@ func newDoctorCmd() *cobra.Command { ansible-playbook, sops, age); the workspace (the credentials, the SSH key, the GitOps repo and its urbex.platform.yaml); with a platform config and a Proxmox token, the node (the API answers, the node, pool, storage, LXC -template and bridge exist, the token has the privileges urbex needs); and, +template and bridge exist, the token has the privileges urbex needs, and the optional +read-only token for the node's metrics); and, with Cloudflare in the config, the token and the zone. It starts with urbex's own version. Each check is ok, warn, fail or skip, with what to do when it isn't ok. It @@ -73,6 +74,11 @@ docs/reference/cli.md in urbforge/urbex.`, p = proxmoxapi.New(cfg.Proxmox.APIURL, creds.ProxmoxToken, cfg.Proxmox.Insecure) } doctor.ProxmoxChecks(ctx, p, cfg, r) + var mon doctor.PermissionsReader + if cfg != nil && creds.ProxmoxMonitorToken != "" { + mon = proxmoxapi.New(cfg.Proxmox.APIURL, creds.ProxmoxMonitorToken, cfg.Proxmox.Insecure) + } + doctor.NodeMetricsChecks(ctx, mon, cfg, r) var cf doctor.CloudflareReader if cfg != nil && creds.CloudflareToken != "" { cf = cloudflare.New(cloudflare.DefaultBaseURL, creds.CloudflareToken, cfg.Cloudflare.AccountID, cfg.Cloudflare.ZoneID) diff --git a/internal/credentials/credentials.go b/internal/credentials/credentials.go index c139254..e2aab4d 100644 --- a/internal/credentials/credentials.go +++ b/internal/credentials/credentials.go @@ -27,6 +27,10 @@ type Credentials struct { ProxmoxToken string `yaml:"proxmoxToken,omitempty"` CloudflareToken string `yaml:"cloudflareToken,omitempty"` AgeKey string `yaml:"ageKey,omitempty"` + // ProxmoxMonitorToken is a read-only (PVEAuditor) token of the whole + // node, "user@realm!name=secret", for the node's metrics in Grafana + // (urbforge/urbex-cli#94); optional, provided by the user. + ProxmoxMonitorToken string `yaml:"proxmoxMonitorToken,omitempty"` // Generated by urbex bootstrap. ForgeAdminPassword string `yaml:"forgeAdminPassword,omitempty"` @@ -58,6 +62,7 @@ func (c *Credentials) fields() []struct { {"URBEX_PROXMOX_TOKEN", &c.ProxmoxToken}, {"URBEX_CLOUDFLARE_TOKEN", &c.CloudflareToken}, {"URBEX_AGE_KEY", &c.AgeKey}, + {"URBEX_PROXMOX_MONITOR_TOKEN", &c.ProxmoxMonitorToken}, {"URBEX_FORGE_ADMIN_PASSWORD", &c.ForgeAdminPassword}, {"URBEX_FORGE_TOKEN", &c.ForgeToken}, {"URBEX_KOMODO_ADMIN_PASSWORD", &c.KomodoAdminPassword}, @@ -309,6 +314,8 @@ var provided = map[string]bool{ "URBEX_PROXMOX_TOKEN": true, "URBEX_CLOUDFLARE_TOKEN": true, "URBEX_AGE_KEY": true, + // The node's metrics. + "URBEX_PROXMOX_MONITOR_TOKEN": true, } // ClearGenerated forgets every credential urbex bootstrap generated - diff --git a/internal/doctor/nodemetrics.go b/internal/doctor/nodemetrics.go new file mode 100644 index 0000000..1f9c45f --- /dev/null +++ b/internal/doctor/nodemetrics.go @@ -0,0 +1,53 @@ +package doctor + +import ( + "context" + "strings" + + "github.com/urbforge/urbex-cli/internal/platform" +) + +// PermissionsReader is what the node metrics check reads with the +// monitoring token (proxmoxapi.Client). +type PermissionsReader interface { + Permissions(ctx context.Context) (map[string]map[string]bool, error) +} + +// nodeMetricsPrivileges are what pve-exporter reads on / : the node's +// status and its storages (PVEAuditor has both). +var nodeMetricsPrivileges = []string{"Sys.Audit", "Datastore.Audit"} + +// NodeMetricsChecks checks the optional read-only token for the Proxmox +// node's metrics (urbforge/urbex-cli#94). m is nil without the token: +// the node metrics are off, which is fine. +func NodeMetricsChecks(ctx context.Context, m PermissionsReader, cfg *platform.Config, r *Report) { + add := func(status, detail, fix string) { + r.add(Check{Group: "proxmox", Name: "node metrics", Status: status, Detail: detail, Fix: fix}) + } + if cfg == nil { + return + } + if m == nil { + add(Skip, "off: no proxmoxMonitorToken, so Grafana shows the LXCs but not the node's CPU, memory and storages", + "optional: create a read-only token (docs/reference/credentials.md#proxmox-monitoring-token), 'urbex-op set proxmoxMonitorToken', then 'urbex bootstrap'") + return + } + perms, err := m.Permissions(ctx) + if err != nil { + add(Fail, "the monitoring token doesn't work: "+firstLine(err.Error()), + "check proxmoxMonitorToken (user@realm!name=secret) and that the token exists on the node") + return + } + var missing []string + for _, p := range nodeMetricsPrivileges { + if !perms["/"][p] { + missing = append(missing, p) + } + } + if len(missing) > 0 { + add(Fail, "the monitoring token lacks "+strings.Join(missing, ", ")+" on /", + "on the node, as root: pveum acl modify / --users --roles PVEAuditor (and --tokens if it is privilege-separated)") + return + } + add(OK, "read-only token with Sys.Audit and Datastore.Audit on /; 'urbex bootstrap' runs pve-exporter", "") +} diff --git a/internal/doctor/nodemetrics_test.go b/internal/doctor/nodemetrics_test.go new file mode 100644 index 0000000..d26bcf8 --- /dev/null +++ b/internal/doctor/nodemetrics_test.go @@ -0,0 +1,49 @@ +package doctor + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/urbforge/urbex-cli/internal/platform" +) + +type fakePerms struct { + perms map[string]map[string]bool + err error +} + +func (f fakePerms) Permissions(context.Context) (map[string]map[string]bool, error) { + return f.perms, f.err +} + +func TestNodeMetricsChecks(t *testing.T) { + ctx := context.Background() + cfg := &platform.Config{} + check := func(m PermissionsReader) Check { + r := &Report{} + NodeMetricsChecks(ctx, m, cfg, r) + if len(r.Checks) != 1 { + t.Fatalf("checks: %+v", r.Checks) + } + return r.Checks[0] + } + if c := check(nil); c.Status != Skip || !strings.Contains(c.Detail, "off") || !strings.Contains(c.Fix, "proxmoxMonitorToken") { + t.Errorf("no token: %+v", c) + } + if c := check(fakePerms{perms: map[string]map[string]bool{"/": {"Sys.Audit": true, "Datastore.Audit": true, "VM.Audit": true}}}); c.Status != OK { + t.Errorf("auditor: %+v", c) + } + if c := check(fakePerms{perms: map[string]map[string]bool{"/pool/urbex": {"Sys.Audit": true}}}); c.Status != Fail || !strings.Contains(c.Detail, "Sys.Audit, Datastore.Audit") || !strings.Contains(c.Fix, "PVEAuditor") { + t.Errorf("pool-only token: %+v", c) + } + if c := check(fakePerms{err: errors.New("401 authentication failure")}); c.Status != Fail || !strings.Contains(c.Detail, "401") { + t.Errorf("bad token: %+v", c) + } + r := &Report{} + NodeMetricsChecks(ctx, nil, nil, r) + if len(r.Checks) != 0 { + t.Error("checked without a config") + } +} diff --git a/internal/platform/catalog.go b/internal/platform/catalog.go index cb382d0..527b682 100644 --- a/internal/platform/catalog.go +++ b/internal/platform/catalog.go @@ -118,6 +118,12 @@ var Catalog = []Service{ {Label: "prometheus", Port: 9090, Exposure: ExposureAccess}, {Label: "prometheus", Port: 9090, Path: "^/api/v1/(write|otlp)", Exposure: ExposureInternal}, }}, + // The Proxmox node's CPU, memory and storages, read from its API with a + // read-only token (urbforge/urbex-cli#94), only when the token is in + // the credentials. Prometheus scrapes it inside the Stack's network: + // it publishes nothing on the LXC, and calls only the node. + {Name: "pve-exporter", Kind: KindPlatform, LXC: "observability", Deploy: DeployGitOps, Provides: []string{"node-metrics"}, + OutboundOnly: true}, {Name: "loki", Kind: KindPlatform, LXC: "observability", Deploy: DeployGitOps, Provides: []string{"logs"}, Endpoints: []Endpoint{ {Label: "loki", Port: 3100, Exposure: ExposureAccess}, diff --git a/platform/stacks/observability/compose.yaml b/platform/stacks/observability/compose.yaml index fef265f..b646541 100644 --- a/platform/stacks/observability/compose.yaml +++ b/platform/stacks/observability/compose.yaml @@ -23,6 +23,27 @@ services: ports: - "9090:9090" + # The Proxmox node's CPU, memory and storages (urbforge/urbex-cli#94), + # with a read-only token: only with the profile pve, which urbex turns + # on in the Stack's environment when the token is in the credentials. + # It reaches the node as pve-node (its address, from the same + # environment), and Prometheus finds it by name in the Stack's network. + pve-exporter: + image: docker.io/prompve/prometheus-pve-exporter:3.12.0@sha256:3541b1f79fdf73200f73ce4863658ae7094a417e6827bd6ea90425c89d61ac12 + profiles: ["pve"] + labels: + urbex.kind: "platform" + urbex.service: "pve-exporter" + restart: unless-stopped + # PVE_USER, PVE_TOKEN_NAME, PVE_TOKEN_VALUE are secrets; PVE_VERIFY_SSL + # is in config.env. + env_file: + - config.env + - path: ${URBEX_SECRETS_FILE:-secrets.env} + required: false + extra_hosts: + - "pve-node:${URBEX_PVE_NODE_IP:-127.0.0.1}" + loki: image: docker.io/grafana/loki:3.7.8 labels: diff --git a/platform/stacks/observability/grafana/dashboards/urbex-resources.json b/platform/stacks/observability/grafana/dashboards/urbex-resources.json index e4b2e4b..18531bd 100644 --- a/platform/stacks/observability/grafana/dashboards/urbex-resources.json +++ b/platform/stacks/observability/grafana/dashboards/urbex-resources.json @@ -591,6 +591,129 @@ "legendFormat": "{{kind}} {{service}} {{project}} {{env}} {{container}} ({{host}})" } ] + }, + { + "id": 11, + "type": "row", + "title": "Proxmox node (with the node's read-only token)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 40, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 12, + "type": "timeseries", + "title": "Node CPU", + "gridPos": { + "x": 0, + "y": 41, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "urbex-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "urbex-prometheus" + }, + "expr": "100 * label_replace(pve_cpu_usage_ratio{id=~\"node/.+\"}, \"node\", \"$1\", \"id\", \"node/(.+)\")", + "legendFormat": "{{node}}" + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Node memory used", + "gridPos": { + "x": 8, + "y": 41, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "urbex-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "urbex-prometheus" + }, + "expr": "label_replace(pve_memory_usage_bytes{id=~\"node/.+\"}, \"node\", \"$1\", \"id\", \"node/(.+)\")", + "legendFormat": "{{node}} used" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "urbex-prometheus" + }, + "expr": "label_replace(pve_memory_size_bytes{id=~\"node/.+\"}, \"node\", \"$1\", \"id\", \"node/(.+)\")", + "legendFormat": "{{node}} total" + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Storage used", + "gridPos": { + "x": 16, + "y": 41, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "urbex-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "urbex-prometheus" + }, + "expr": "label_replace(100 * pve_disk_usage_bytes{id=~\"storage/.+\"} / (pve_disk_size_bytes{id=~\"storage/.+\"} > 0), \"storage\", \"$1\", \"id\", \"storage/(.+)\")", + "legendFormat": "{{storage}}" + } + ] } ] } diff --git a/platform/stacks/observability/grafana/provisioning/alerting/urbex-resources.yml b/platform/stacks/observability/grafana/provisioning/alerting/urbex-resources.yml index 11a9431..0547a1b 100644 --- a/platform/stacks/observability/grafana/provisioning/alerting/urbex-resources.yml +++ b/platform/stacks/observability/grafana/provisioning/alerting/urbex-resources.yml @@ -101,3 +101,91 @@ groups: annotations: summary: '{{ $labels.host }} has {{ printf "%.0f" $values.B.Value }}% of its memory available.' description: Below 10% for ten minutes. Its services may be killed when it runs out. Give the machine more memory (for an application, resources.memory in urbex.yaml), or find what grows in the Urbex resources dashboard. + # The Proxmox node itself (urbforge/urbex-cli#94), from pve-exporter, + # only with the node's read-only token: its memory - guests outside + # urbex and ZFS's ARC included, as Proxmox's summary counts it - and + # each storage. Without the token there is no series, and no alert. + - uid: urbex-node-memory-high + title: The Proxmox node is running out of memory + condition: C + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: urbex-prometheus + model: + refId: A + instant: true + expr: 'label_replace(100 * pve_memory_usage_bytes{id=~"node/.+"} / pve_memory_size_bytes{id=~"node/.+"}, "node", "$1", "id", "node/(.+)")' + - refId: B + datasourceUid: __expr__ + model: { refId: B, type: reduce, expression: A, reducer: last } + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: [{ evaluator: { type: gt, params: [90] } }] + noDataState: OK + execErrState: Error + for: 10m + labels: { kind: platform, service: proxmox, severity: warning } + annotations: + summary: 'The Proxmox node {{ $labels.node }} has {{ printf "%.0f" $values.B.Value }}% of its memory used.' + description: Above 90% for ten minutes. Proxmox may stop guests when it runs out. Check the node's Summary in Proxmox, give LXCs less memory, or cap ZFS's ARC. + - uid: urbex-node-storage-filling + title: A Proxmox storage is filling up + condition: C + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: urbex-prometheus + model: + refId: A + instant: true + expr: 'label_replace(label_replace(100 * pve_disk_usage_bytes{id=~"storage/.+"} / (pve_disk_size_bytes{id=~"storage/.+"} > 0), "node", "$1", "id", "storage/([^/]+)/.+"), "storage", "$1", "id", "storage/[^/]+/(.+)")' + - refId: B + datasourceUid: __expr__ + model: { refId: B, type: reduce, expression: A, reducer: last } + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: [{ evaluator: { type: gt, params: [85] } }] + noDataState: OK + execErrState: Error + for: 5m + labels: { kind: platform, service: proxmox, severity: warning } + annotations: + summary: 'Storage {{ $labels.storage }} on {{ $labels.node }} is {{ printf "%.0f" $values.B.Value }}% full.' + description: Above 85%. New LXCs, volumes and backups need room there. Free space, or move guests to another storage. + - uid: urbex-node-storage-full + title: A Proxmox storage is nearly full + condition: C + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: urbex-prometheus + model: + refId: A + instant: true + expr: 'label_replace(label_replace(100 * pve_disk_usage_bytes{id=~"storage/.+"} / (pve_disk_size_bytes{id=~"storage/.+"} > 0), "node", "$1", "id", "storage/([^/]+)/.+"), "storage", "$1", "id", "storage/[^/]+/(.+)")' + - refId: B + datasourceUid: __expr__ + model: { refId: B, type: reduce, expression: A, reducer: last } + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: [{ evaluator: { type: gt, params: [95] } }] + noDataState: OK + execErrState: Error + for: 5m + labels: { kind: platform, service: proxmox, severity: critical } + annotations: + summary: 'Storage {{ $labels.storage }} on {{ $labels.node }} is {{ printf "%.0f" $values.B.Value }}% full.' + description: Above 95%. Guests on it can stop writing. Free space now. diff --git a/platform/stacks/observability/prometheus.yml b/platform/stacks/observability/prometheus.yml index e6b3c54..30f2eb0 100644 --- a/platform/stacks/observability/prometheus.yml +++ b/platform/stacks/observability/prometheus.yml @@ -6,6 +6,28 @@ scrape_configs: - job_name: prometheus static_configs: - targets: ["localhost:9090"] + # The Proxmox node, through pve-exporter (urbforge/urbex-cli#94). Found + # by name in the Stack's network: without the node's token the exporter + # doesn't run, the name doesn't resolve, and there is no target. + - job_name: pve + scrape_interval: 60s + scrape_timeout: 30s + metrics_path: /pve + params: + target: [pve-node] + cluster: ["1"] + node: ["1"] + dns_sd_configs: + - names: [pve-exporter] + type: A + port: 9221 + refresh_interval: 60s + # The token sees the whole node: keep the node and its storages, never + # the guests (urbex's own LXCs report through their agents). + metric_relabel_configs: + - source_labels: [id] + regex: "(node|storage)/.+" + action: keep # The LXCs' resource metrics are pushed by their telemetry agents (remote # write), labelled kind/service/host (and project/env for apps). Scraping # the services' own metrics (urbex.yaml's observability.metrics) is not From 1465eeba2e1ed45823b3a3bf9505d9ed45f6c83d Mon Sep 17 00:00:00 2001 From: Teg79 Date: Thu, 19 Nov 2026 21:08:35 +0100 Subject: [PATCH 2/2] fix(doctor): say when the monitoring token is only its secret, without the token's id Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182yxsfwRowwnxCfgrQwjnA --- internal/cli/doctor.go | 2 +- internal/doctor/nodemetrics.go | 11 ++++++++++- internal/doctor/nodemetrics_test.go | 10 ++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 686685f..1125e46 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -78,7 +78,7 @@ docs/reference/cli.md in urbforge/urbex.`, if cfg != nil && creds.ProxmoxMonitorToken != "" { mon = proxmoxapi.New(cfg.Proxmox.APIURL, creds.ProxmoxMonitorToken, cfg.Proxmox.Insecure) } - doctor.NodeMetricsChecks(ctx, mon, cfg, r) + doctor.NodeMetricsChecks(ctx, mon, creds.ProxmoxMonitorToken, cfg, r) var cf doctor.CloudflareReader if cfg != nil && creds.CloudflareToken != "" { cf = cloudflare.New(cloudflare.DefaultBaseURL, creds.CloudflareToken, cfg.Cloudflare.AccountID, cfg.Cloudflare.ZoneID) diff --git a/internal/doctor/nodemetrics.go b/internal/doctor/nodemetrics.go index 1f9c45f..05c99b4 100644 --- a/internal/doctor/nodemetrics.go +++ b/internal/doctor/nodemetrics.go @@ -2,6 +2,7 @@ package doctor import ( "context" + "regexp" "strings" "github.com/urbforge/urbex-cli/internal/platform" @@ -15,12 +16,15 @@ type PermissionsReader interface { // nodeMetricsPrivileges are what pve-exporter reads on / : the node's // status and its storages (PVEAuditor has both). +// proxmoxToken is an API token as urbex stores it: user@realm!name=secret. +var proxmoxToken = regexp.MustCompile(`^[^@!=\s]+@[^@!=\s]+![^@!=\s]+=\S+$`) + var nodeMetricsPrivileges = []string{"Sys.Audit", "Datastore.Audit"} // NodeMetricsChecks checks the optional read-only token for the Proxmox // node's metrics (urbforge/urbex-cli#94). m is nil without the token: // the node metrics are off, which is fine. -func NodeMetricsChecks(ctx context.Context, m PermissionsReader, cfg *platform.Config, r *Report) { +func NodeMetricsChecks(ctx context.Context, m PermissionsReader, token string, cfg *platform.Config, r *Report) { add := func(status, detail, fix string) { r.add(Check{Group: "proxmox", Name: "node metrics", Status: status, Detail: detail, Fix: fix}) } @@ -32,6 +36,11 @@ func NodeMetricsChecks(ctx context.Context, m PermissionsReader, cfg *platform.C "optional: create a read-only token (docs/reference/credentials.md#proxmox-monitoring-token), 'urbex-op set proxmoxMonitorToken', then 'urbex bootstrap'") return } + if !proxmoxToken.MatchString(token) { + add(Fail, "proxmoxMonitorToken isn't a whole token: it needs the token's id before its secret", + "store it again as urbex-monitor@pve!metrics= ('urbex-op set proxmoxMonitorToken'): the id is what 'pveum user token add' names, the secret its value") + return + } perms, err := m.Permissions(ctx) if err != nil { add(Fail, "the monitoring token doesn't work: "+firstLine(err.Error()), diff --git a/internal/doctor/nodemetrics_test.go b/internal/doctor/nodemetrics_test.go index d26bcf8..f713f4e 100644 --- a/internal/doctor/nodemetrics_test.go +++ b/internal/doctor/nodemetrics_test.go @@ -21,9 +21,10 @@ func (f fakePerms) Permissions(context.Context) (map[string]map[string]bool, err func TestNodeMetricsChecks(t *testing.T) { ctx := context.Background() cfg := &platform.Config{} + token := "urbex-monitor@pve!metrics=00000000-1111-2222-3333-444444444444" check := func(m PermissionsReader) Check { r := &Report{} - NodeMetricsChecks(ctx, m, cfg, r) + NodeMetricsChecks(ctx, m, token, cfg, r) if len(r.Checks) != 1 { t.Fatalf("checks: %+v", r.Checks) } @@ -41,8 +42,13 @@ func TestNodeMetricsChecks(t *testing.T) { if c := check(fakePerms{err: errors.New("401 authentication failure")}); c.Status != Fail || !strings.Contains(c.Detail, "401") { t.Errorf("bad token: %+v", c) } + // Only the secret, as 'pveum user token add' prints it on its own line. + token = "00000000-1111-2222-3333-444444444444" + if c := check(fakePerms{err: errors.New("401")}); c.Status != Fail || !strings.Contains(c.Detail, "token's id") || !strings.Contains(c.Fix, "urbex-monitor@pve!metrics=") { + t.Errorf("secret only: %+v", c) + } r := &Report{} - NodeMetricsChecks(ctx, nil, nil, r) + NodeMetricsChecks(ctx, nil, "", nil, r) if len(r.Checks) != 0 { t.Error("checked without a config") }